Build Help documentation during CMake build using Sphinx

Add a Utilities/Sphinx directory to hold CMake build code to run the
Sphinx (sphinx-doc.org) documentation generation tool.  Create a
CMakeLists.txt file there capable of building either as a subdirectory
of the main CMake build, or as a standalone documentation build.

Add cache options SPHINX_MAN and SPHINX_HTML to select output formats
and SPHINX_EXECUTABLE to specify the sphinx-build executable.  Add
bootstrap options --sphix-man and --sphinx-html to select output formats
and --sphinx-build=<sb> to specify the sphinx-build executable.

Create a "conf.py.in" file to configure_file into "conf.py" to tell
sphinx-build how to build our documents.  Create a "cmake.py" Sphinx
extension module defining:

* The "cmake-module" directive used in Help/module/*.rst files to
  scan .rst markup from the corresponding Modules/*.cmake file.

* A Sphinx domain called "cmake" defining documentation object types
  for CMake Help/<type> directories: command, generator, manual,
  module, policy, prop_*, and variable.  Add a "role" for each type
  to perform cross-references.  Teach the roles to treat "<XYZ>"
  as placeholders instead of explicit targets if not preceded by
  a space.  Add cmake domain directives to define command and
  variable objects explicitly in .rst file content.  This will
  allow modules to define their own commands and variables and
  have them indexed and linkable.

* A Sphinx document transform that converts Help/<type>/*.rst documents
  into cmake domain objects of the corresponding <type> and adds index
  entries for them.  This will automatically index all CMake documentation
  objects and provide cross-reference targets for them with no special
  markup in the .rst files.

Utilities/CMakeLists.txt

@@ -10,3 +10,5 @@
 # See the License for more information.
 #=============================================================================
 subdirs(Doxygen KWStyle)
+
+add_subdirectory(Sphinx)

Utilities/Sphinx/.gitignore

@@ -0,0 +1 @@
+*.pyc

Utilities/Sphinx/CMakeLists.txt

@@ -0,0 +1,106 @@
+#=============================================================================
+# CMake - Cross Platform Makefile Generator
+# Copyright 2000-2013 Kitware, Inc., Insight Software Consortium
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+if(NOT CMake_SOURCE_DIR)
+  set(CMakeHelp_STANDALONE 1)
+  cmake_minimum_required(VERSION 2.8.2 FATAL_ERROR)
+  set(CMAKE_LEGACY_CYGWIN_WIN32 0) # Remove when CMake >= 2.8.4 is required
+  get_filename_component(tmp "${CMAKE_CURRENT_SOURCE_DIR}" PATH)
+  get_filename_component(CMake_SOURCE_DIR "${tmp}" PATH)
+  include(${CMake_SOURCE_DIR}/Source/CMakeVersionCompute.cmake)
+  include(${CMake_SOURCE_DIR}/Source/CMakeInstallDestinations.cmake)
+  unset(CMAKE_DATA_DIR)
+  unset(CMAKE_DATA_DIR CACHE)
+endif()
+project(CMakeHelp NONE)
+
+option(SPHINX_MAN "Build man pages with Sphinx" OFF)
+option(SPHINX_HTML "Build html help with Sphinx" OFF)
+find_program(SPHINX_EXECUTABLE
+  NAMES sphinx-build
+  DOC "Sphinx Documentation Builder (sphinx-doc.org)"
+  )
+
+if(NOT SPHINX_MAN AND NOT SPHINX_HTML)
+  return()
+elseif(NOT SPHINX_EXECUTABLE)
+  message(FATAL_ERROR "SPHINX_EXECUTABLE (sphinx-build) is not found!")
+endif()
+
+set(conf_path "${CMAKE_CURRENT_SOURCE_DIR}")
+set(conf_copyright "2000-2013 Kitware, Inc.")
+set(conf_version "${CMake_MAJOR_VERSION}.${CMake_MINOR_VERSION}.${CMake_PATCH_VERSION}")
+set(conf_release "${CMake_VERSION}")
+configure_file(conf.py.in conf.py @ONLY)
+
+set(doc_formats "")
+if(SPHINX_HTML)
+  list(APPEND doc_formats html)
+endif()
+if(SPHINX_MAN)
+  list(APPEND doc_formats man)
+endif()
+
+set(doc_format_outputs "")
+set(doc_format_last "")
+foreach(format ${doc_formats})
+  set(doc_format_output "doc_format_${format}")
+  set(doc_format_log "build-${format}.log")
+  add_custom_command(
+    OUTPUT ${doc_format_output}
+    COMMAND ${SPHINX_EXECUTABLE}
+            -c ${CMAKE_CURRENT_BINARY_DIR}
+            -d ${CMAKE_CURRENT_BINARY_DIR}/doctrees
+            -b ${format}
+            ${CMake_SOURCE_DIR}/Help
+            ${CMAKE_CURRENT_BINARY_DIR}/${format}
+            > ${doc_format_log} # log stdout, pass stderr
+    DEPENDS ${doc_format_last}
+    COMMENT "sphinx-build ${format}: see Utilities/Sphinx/${doc_format_log}"
+    VERBATIM
+    )
+  set_property(SOURCE ${doc_format_output} PROPERTY SYMBOLIC 1)
+  list(APPEND doc_format_outputs ${doc_format_output})
+  set(doc_format_last ${doc_format_output})
+endforeach()
+
+add_custom_target(documentation ALL DEPENDS ${doc_format_outputs})
+
+foreach(t
+    cmake
+    ccmake
+    cmake-gui
+    cpack
+    ctest
+    )
+  if(TARGET ${t})
+    # Build documentation after main executables.
+    add_dependencies(documentation ${t})
+  endif()
+endforeach()
+
+if(SPHINX_MAN)
+  file(GLOB man_rst RELATIVE ${CMake_SOURCE_DIR}/Help/manual
+    ${CMake_SOURCE_DIR}/Help/manual/*.[1-9].rst)
+  foreach(m ${man_rst})
+    if("x${m}" MATCHES "^x(.+)\\.([1-9])\\.rst$")
+      set(name "${CMAKE_MATCH_1}")
+      set(sec "${CMAKE_MATCH_2}")
+      install(FILES ${CMAKE_CURRENT_BINARY_DIR}/man/${name}.${sec}
+              DESTINATION ${CMAKE_MAN_DIR}/man${sec})
+    endif()
+  endforeach()
+endif()
+
+if(SPHINX_HTML)
+  install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html
+          DESTINATION ${CMAKE_DOC_DIR})
+endif()

Utilities/Sphinx/cmake.py

@@ -0,0 +1,282 @@
+#=============================================================================
+# CMake - Cross Platform Makefile Generator
+# Copyright 2000-2013 Kitware, Inc., Insight Software Consortium
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+import os
+import re
+
+from docutils.parsers.rst import Directive, directives
+from docutils.transforms import Transform
+from docutils.utils.error_reporting import SafeString, ErrorString
+from docutils import io, nodes
+
+from sphinx.directives import ObjectDescription
+from sphinx.domains import Domain, ObjType
+from sphinx.roles import XRefRole
+from sphinx.util.nodes import make_refnode
+from sphinx import addnodes
+
+class CMakeModule(Directive):
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    option_spec = {'encoding': directives.encoding}
+
+    def __init__(self, *args, **keys):
+        self.re_start = re.compile(r'^#\[(?P<eq>=*)\[\.rst:$')
+        self.re_end = re.compile(r'^#?\](?P<eq>=*)\]$')
+        Directive.__init__(self, *args, **keys)
+
+    def run(self):
+        settings = self.state.document.settings
+        if not settings.file_insertion_enabled:
+            raise self.warning('"%s" directive disabled.' % self.name)
+
+        env = self.state.document.settings.env
+        rel_path, path = env.relfn2path(self.arguments[0])
+        path = os.path.normpath(path)
+        encoding = self.options.get('encoding', settings.input_encoding)
+        e_handler = settings.input_encoding_error_handler
+        try:
+            settings.record_dependencies.add(path)
+            f = io.FileInput(source_path=path, encoding=encoding,
+                             error_handler=e_handler)
+        except UnicodeEncodeError, error:
+            raise self.severe('Problems with "%s" directive path:\n'
+                              'Cannot encode input file path "%s" '
+                              '(wrong locale?).' %
+                              (self.name, SafeString(path)))
+        except IOError, error:
+            raise self.severe('Problems with "%s" directive path:\n%s.' %
+                      (self.name, ErrorString(error)))
+        raw_lines = f.read().splitlines()
+        f.close()
+        rst = None
+        lines = []
+        for line in raw_lines:
+            if line == '#.rst:':
+                rst = '#'
+                line = ''
+            elif rst == '#':
+                if line == '#' or line[:2] == '# ':
+                    line = line[2:]
+                else:
+                    rst = None
+                    line = ''
+            else:
+                line = ''
+            lines.append(line)
+        self.state_machine.insert_input(lines, path)
+        return []
+
+class _cmake_index_entry:
+    def __init__(self, desc):
+        self.desc = desc
+
+    def __call__(self, title, targetid):
+        return ('pair', u'%s ; %s' % (self.desc, title), targetid, 'main')
+
+_cmake_index_objs = {
+    'command':    _cmake_index_entry('command'),
+    'generator':  _cmake_index_entry('generator'),
+    'manual':     _cmake_index_entry('manual'),
+    'module':     _cmake_index_entry('module'),
+    'policy':     _cmake_index_entry('policy'),
+    'prop_cache': _cmake_index_entry('cache property'),
+    'prop_dir':   _cmake_index_entry('directory property'),
+    'prop_gbl':   _cmake_index_entry('global property'),
+    'prop_sf':    _cmake_index_entry('source file property'),
+    'prop_test':  _cmake_index_entry('test property'),
+    'prop_tgt':   _cmake_index_entry('target property'),
+    'variable':   _cmake_index_entry('variable'),
+    }
+
+def _cmake_object_inventory(env, document, line, objtype, targetid):
+    inv = env.domaindata['cmake']['objects']
+    if targetid in inv:
+        document.reporter.warning(
+            'CMake object "%s" also described in "%s".' %
+            (targetid, env.doc2path(inv[targetid][0])), line=line)
+    inv[targetid] = (env.docname, objtype)
+
+class CMakeTransform(Transform):
+
+    # Run this transform early since we insert nodes we want
+    # treated as if they were written in the documents.
+    default_priority = 210
+
+    def __init__(self, document, startnode):
+        Transform.__init__(self, document, startnode)
+        self.titles = {}
+
+    def parse_title(self, docname):
+        """Parse a document title as the first line starting in [A-Za-z0-9<]
+           or fall back to the document basename if no such line exists.
+           The cmake --help-*-list commands also depend on this convention.
+           Return the title or False if the document file does not exist.
+        """
+        env = self.document.settings.env
+        title = self.titles.get(docname)
+        if title is None:
+            fname = os.path.join(env.srcdir, docname+'.rst')
+            try:
+                f = open(fname, 'r')
+            except IOError:
+                title = False
+            else:
+                for line in f:
+                    if len(line) > 0 and (line[0].isalnum() or line[0] == '<'):
+                        title = line.rstrip()
+                        break
+                f.close()
+                if title is None:
+                    title = os.path.basename(docname)
+            self.titles[docname] = title
+        return title
+
+    def apply(self):
+        env = self.document.settings.env
+
+        # Treat some documents as cmake domain objects.
+        objtype, sep, tail = env.docname.rpartition('/')
+        make_index_entry = _cmake_index_objs.get(objtype)
+        if make_index_entry:
+            title = self.parse_title(env.docname)
+            # Insert the object link target.
+            targetid = '%s:%s' % (objtype, title)
+            targetnode = nodes.target('', '', ids=[targetid])
+            self.document.insert(0, targetnode)
+            # Insert the object index entry.
+            indexnode = addnodes.index()
+            indexnode['entries'] = [make_index_entry(title, targetid)]
+            self.document.insert(0, indexnode)
+            # Add to cmake domain object inventory
+            _cmake_object_inventory(env, self.document, 1, objtype, targetid)
+
+class CMakeObject(ObjectDescription):
+
+    def handle_signature(self, sig, signode):
+        # called from sphinx.directives.ObjectDescription.run()
+        signode += addnodes.desc_name(sig, sig)
+        return sig
+
+    def add_target_and_index(self, name, sig, signode):
+        targetid = '%s:%s' % (self.objtype, name)
+        if targetid not in self.state.document.ids:
+            signode['names'].append(targetid)
+            signode['ids'].append(targetid)
+            signode['first'] = (not self.names)
+            self.state.document.note_explicit_target(signode)
+            _cmake_object_inventory(self.env, self.state.document,
+                                    self.lineno, self.objtype, targetid)
+
+        make_index_entry = _cmake_index_objs.get(self.objtype)
+        if make_index_entry:
+            self.indexnode['entries'].append(make_index_entry(name, targetid))
+
+class CMakeXRefRole(XRefRole):
+
+    # See sphinx.util.nodes.explicit_title_re; \x00 escapes '<'.
+    _re = re.compile(r'^(.+?)(\s*)(?<!\x00)<(.*?)>$', re.DOTALL)
+    _re_sub = re.compile(r'^([^()\s]+)\s*\(([^()]*)\)$', re.DOTALL)
+
+    def __call__(self, typ, rawtext, text, *args, **keys):
+        # Translate CMake command cross-references of the form:
+        #  `command_name(SUB_COMMAND)`
+        # to have an explicit target:
+        #  `command_name(SUB_COMMAND) <command_name>`
+        if typ == 'cmake:command':
+            m = CMakeXRefRole._re_sub.match(text)
+            if m:
+                text = '%s <%s>' % (text, m.group(1))
+        # CMake cross-reference targets frequently contain '<' so escape
+        # any explicit `<target>` with '<' not preceded by whitespace.
+        while True:
+            m = CMakeXRefRole._re.match(text)
+            if m and len(m.group(2)) == 0:
+                text = '%s\x00<%s>' % (m.group(1), m.group(3))
+            else:
+                break
+        return XRefRole.__call__(self, typ, rawtext, text, *args, **keys)
+
+class CMakeDomain(Domain):
+    """CMake domain."""
+    name = 'cmake'
+    label = 'CMake'
+    object_types = {
+        'command':    ObjType('command',    'command'),
+        'generator':  ObjType('generator',  'generator'),
+        'variable':   ObjType('variable',   'variable'),
+        'module':     ObjType('module',     'module'),
+        'policy':     ObjType('policy',     'policy'),
+        'prop_cache': ObjType('prop_cache', 'prop_cache'),
+        'prop_dir':   ObjType('prop_dir',   'prop_dir'),
+        'prop_gbl':   ObjType('prop_gbl',   'prop_gbl'),
+        'prop_sf':    ObjType('prop_sf',    'prop_sf'),
+        'prop_test':  ObjType('prop_test',  'prop_test'),
+        'prop_tgt':   ObjType('prop_tgt',   'prop_tgt'),
+        'manual':     ObjType('manual',     'manual'),
+    }
+    directives = {
+        'command':    CMakeObject,
+        'variable':   CMakeObject,
+        # Other object types cannot be created except by the CMakeTransform
+        # 'generator':  CMakeObject,
+        # 'module':     CMakeObject,
+        # 'policy':     CMakeObject,
+        # 'prop_cache': CMakeObject,
+        # 'prop_dir':   CMakeObject,
+        # 'prop_gbl':   CMakeObject,
+        # 'prop_sf':    CMakeObject,
+        # 'prop_test':  CMakeObject,
+        # 'prop_tgt':   CMakeObject,
+        # 'manual':     CMakeObject,
+    }
+    roles = {
+        'command':    CMakeXRefRole(fix_parens = True, lowercase = True),
+        'generator':  CMakeXRefRole(),
+        'variable':   CMakeXRefRole(),
+        'module':     CMakeXRefRole(),
+        'policy':     CMakeXRefRole(),
+        'prop_cache': CMakeXRefRole(),
+        'prop_dir':   CMakeXRefRole(),
+        'prop_gbl':   CMakeXRefRole(),
+        'prop_sf':    CMakeXRefRole(),
+        'prop_test':  CMakeXRefRole(),
+        'prop_tgt':   CMakeXRefRole(),
+        'manual':     CMakeXRefRole(),
+    }
+    initial_data = {
+        'objects': {},  # fullname -> docname, objtype
+    }
+
+    def clear_doc(self, docname):
+        for fullname, (fn, _) in self.data['objects'].items():
+            if fn == docname:
+                del self.data['objects'][fullname]
+
+    def resolve_xref(self, env, fromdocname, builder,
+                     typ, target, node, contnode):
+        targetid = '%s:%s' % (typ, target)
+        obj = self.data['objects'].get(targetid)
+        if obj is None:
+            # TODO: warn somehow?
+            return None
+        return make_refnode(builder, fromdocname, obj[0], targetid,
+                            contnode, target)
+
+    def get_objects(self):
+        for refname, (docname, type) in self.data['objects'].iteritems():
+            yield (refname, refname, type, docname, refname, 1)
+
+def setup(app):
+    app.add_directive('cmake-module', CMakeModule)
+    app.add_transform(CMakeTransform)
+    app.add_domain(CMakeDomain)

Utilities/Sphinx/conf.py.in

@@ -0,0 +1,47 @@
+#=============================================================================
+# CMake - Cross Platform Makefile Generator
+# Copyright 2000-2013 Kitware, Inc., Insight Software Consortium
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+import sys
+
+sys.path.insert(0, r'@conf_path@')
+
+source_suffix = '.rst'
+master_doc = 'index'
+project = 'CMake'
+copyright = '@conf_copyright@'
+
+version = '@conf_version@' # feature version
+release = '@conf_release@' # full version string
+
+primary_domain = 'cmake'
+
+exclude_patterns = []
+
+extensions = ['cmake']
+
+man_pages = [
+    ('manual/cmake.1', 'cmake', 'CMake Command-Line Reference', [], 1),
+    ('manual/ccmake.1', 'ccmake', 'CMake Curses Dialog Command-Line Reference', [], 1),
+    ('manual/cmake-gui.1', 'cmake-gui', 'CMake GUI Command-Line  Reference', [], 1),
+    ('manual/cpack.1', 'cpack', 'CPack Command-Line Reference', [], 1),
+    ('manual/ctest.1', 'ctest', 'CTest Command-Line Reference', [], 1),
+    ('manual/cmake-commands.7', 'cmake-commands', 'CMake Language Command Reference', [], 7),
+    ('manual/cmake-generators.7', 'cmake-generators', 'CMake Generators Reference', [], 7),
+    ('manual/cmake-modules.7', 'cmake-modules', 'CMake Modules Reference', [], 7),
+    ('manual/cmake-policies.7', 'cmake-policies', 'CMake Policies Reference', [], 7),
+    ('manual/cmake-properties.7', 'cmake-properties', 'CMake Properties Reference', [], 7),
+    ('manual/cmake-variables.7', 'cmake-variables', 'CMake Variables Reference', [], 7),
+]
+man_show_urls = False
+
+html_show_sourcelink = True
+html_static_path = ['@conf_path@/static']
+html_style = 'cmake.css'

Utilities/Sphinx/static/cmake.css

@@ -0,0 +1,8 @@
+/* Import the Sphinx theme style.  */
+@import url("default.css");
+
+/* Wrap sidebar content even within words so that long
+   document names do not escape sidebar borders.  */
+div.sphinxsidebarwrapper {
+  word-wrap: break-word;
+}

bootstrap

@@ -66,6 +66,9 @@ cmake_init_file=""
 cmake_bootstrap_system_libs=""
 cmake_bootstrap_qt_gui=""
 cmake_bootstrap_qt_qmake=""
+cmake_sphinx_man=""
+cmake_sphinx_html=""
+cmake_sphinx_build=""
 
 # Determine whether this is a Cygwin environment.
 if echo "${cmake_system}" | grep CYGWIN >/dev/null 2>&1; then
@@ -375,6 +378,10 @@ Configuration:
   --no-qt-gui             do not build the Qt-based GUI (default)
   --qt-qmake=<qmake>      use <qmake> as the qmake executable to find Qt
 
+  --sphinx-man            build man pages with Sphinx
+  --sphinx-html           build html help with Sphinx
+  --sphinx-build=<sb>     use <sb> as the sphinx-build executable
+
 Directory and file names:
   --prefix=PREFIX         install files in tree rooted at PREFIX
                           ['"${cmake_default_prefix}"']
@@ -606,6 +613,9 @@ while test $# != 0; do
   --qt-gui) cmake_bootstrap_qt_gui="1" ;;
   --no-qt-gui) cmake_bootstrap_qt_gui="0" ;;
   --qt-qmake=*) cmake_bootstrap_qt_qmake=`cmake_arg "$1"` ;;
+  --sphinx-man) cmake_sphinx_man="1" ;;
+  --sphinx-html) cmake_sphinx_html="1" ;;
+  --sphinx-build=*) cmake_sphinx_build=`cmake_arg "$1"` ;;
   --help) cmake_usage ;;
   --version) cmake_version_display ; exit 2 ;;
   --verbose) cmake_verbose=TRUE ;;
@@ -1563,6 +1573,21 @@ if [ "x${cmake_bootstrap_qt_qmake}" != "x" ]; then
 set (QT_QMAKE_EXECUTABLE "'"${cmake_bootstrap_qt_qmake}"'" CACHE FILEPATH "Location of Qt qmake" FORCE)
 ' >> "${cmake_bootstrap_dir}/InitialCacheFlags.cmake"
 fi
+if [ "x${cmake_sphinx_man}" != "x" ]; then
+  echo '
+set (SPHINX_MAN "'"${cmake_sphinx_man}"'" CACHE FILEPATH "Build man pages with Sphinx" FORCE)
+' >> "${cmake_bootstrap_dir}/InitialCacheFlags.cmake"
+fi
+if [ "x${cmake_sphinx_html}" != "x" ]; then
+  echo '
+set (SPHINX_HTML "'"${cmake_sphinx_html}"'" CACHE FILEPATH "Build html help with Sphinx" FORCE)
+' >> "${cmake_bootstrap_dir}/InitialCacheFlags.cmake"
+fi
+if [ "x${cmake_sphinx_build}" != "x" ]; then
+  echo '
+set (SPHINX_EXECUTABLE "'"${cmake_sphinx_build}"'" CACHE FILEPATH "Location of Qt sphinx-build" FORCE)
+' >> "${cmake_bootstrap_dir}/InitialCacheFlags.cmake"
+fi
 
 # Add user-specified settings.  Handle relative-path case for
 # specification of cmake_init_file.