Bug 1275409: move templates to taskgraph.util; r=wcosta

MozReview-Commit-ID: 3vdnm20W4OD

--HG--
rename : taskcluster/taskgraph/test/test_util.py => taskcluster/taskgraph/test/test_util_docker.py
rename : testing/taskcluster/tests/test_templates.py => taskcluster/taskgraph/test/test_util_templates.py
rename : taskcluster/taskgraph/util.py => taskcluster/taskgraph/util/__init__.py
rename : testing/taskcluster/taskcluster_graph/templates.py => taskcluster/taskgraph/util/templates.py
extra : rebase_source : 6d098d87e715b82c0dcd5bf03beb7646bbd50fe2

taskcluster/docs/index.rst

@@ -17,4 +17,5 @@ than you might suppose!  This implementation supports:
     taskgraph
     parameters
     attributes
+    yaml-templates
     old

taskcluster/docs/yaml-templates.rst

@@ -0,0 +1,45 @@
+Task Definition YAML Templates
+==============================
+
+Many kinds of tasks are described using YAML files.  These files allow some
+limited forms of inheritance and template substitution as well as the usual
+YAML features, as described below.
+
+Please use these features sparingly.  In many cases, it is better to add a
+feature to the implementation of a task kind rather than add complexity to the
+YAML files.
+
+Inheritance
+-----------
+
+One YAML file can "inherit" from another by including a top-level ``$inherits``
+key.  That key specifies the parent file in ``from``, and optionally a
+collection of variables in ``variables``.  For example:
+
+.. code-block:: yaml
+
+    $inherits:
+      from: 'tasks/builds/base_linux32.yml'
+      variables:
+        build_name: 'linux32'
+        build_type: 'dbg'
+
+Inheritance proceeds as follows: First, the child document has its template
+substitutions performed and is parsed as YAML.  Then, the parent document is
+parsed, with substitutions specified by ``variables`` added to the template
+substitutions.  Finally, the child document is merged with the parent.
+
+To merge two JSON objects (dictionaries), each value is merged individually.
+Lists are merged by concatenating the lists from the parent and child
+documents.  Atomic values (strings, numbers, etc.) are merged by preferring the
+child document's value.
+
+Substitution
+------------
+
+Each document is expanded using the PyStache template engine before it is
+parsed as YAML.  The parameters for this expansion are specific to the task
+kind.
+
+Simple value substitution looks like ``{{variable}}``.  Function calls look
+like ``{{#function}}argument{{/function}}``.

taskcluster/taskgraph/kind/docker_image.py

@@ -14,10 +14,10 @@ import time
 
 from . import base
 from ..types import Task
-from taskgraph.util import docker_image
+from taskgraph.util.docker import docker_image
 import taskcluster_graph.transform.routes as routes_transform
 import taskcluster_graph.transform.treeherder as treeherder_transform
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
 from taskcluster_graph.from_now import (
     json_time_from_now,
     current_json_time,

taskcluster/taskgraph/kind/legacy.py

@@ -32,9 +32,9 @@ from taskcluster_graph.from_now import (
     json_time_from_now,
     current_json_time,
 )
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
 import taskcluster_graph.build_task
-from taskgraph.util import docker_image
+from taskgraph.util.docker import docker_image
 
 # TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
 # actual taskIds at the very last minute, in get_task_definition

taskcluster/taskgraph/test/test_util_docker.py

@@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
-from ..util import docker_image, DOCKER_ROOT
+from ..util.docker import docker_image, DOCKER_ROOT
 from mozunit import main, MockedOpen
 
 

taskcluster/taskgraph/test/test_util_templates.py

@@ -0,0 +1,189 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+import unittest
+import mozunit
+import textwrap
+from taskgraph.util.templates import (
+    Templates,
+    TemplatesException
+)
+
+files = {}
+files['/fixtures/circular.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'circular_ref.yml'
+      variables:
+        woot: 'inherit'
+    """)
+
+files['/fixtures/inherit.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'templates.yml'
+      variables:
+        woot: 'inherit'
+    """)
+
+files['/fixtures/extend_child.yml'] = textwrap.dedent("""\
+    list: ['1', '2', '3']
+    was_list: ['1']
+    obj:
+      level: 1
+      deeper:
+        woot: 'bar'
+        list: ['baz']
+    """)
+
+files['/fixtures/circular_ref.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'circular.yml'
+    """)
+
+files['/fixtures/child_pass.yml'] = textwrap.dedent("""\
+    values:
+      - {{a}}
+      - {{b}}
+      - {{c}}
+    """)
+
+files['/fixtures/inherit_pass.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'child_pass.yml'
+      variables:
+        a: 'a'
+        b: 'b'
+        c: 'c'
+    """)
+
+files['/fixtures/deep/2.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/1.yml
+
+    """)
+
+files['/fixtures/deep/3.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/2.yml
+
+    """)
+
+files['/fixtures/deep/4.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/3.yml
+    """)
+
+files['/fixtures/deep/1.yml'] = textwrap.dedent("""\
+    variable: {{value}}
+    """)
+
+files['/fixtures/simple.yml'] = textwrap.dedent("""\
+    is_simple: true
+    """)
+
+files['/fixtures/templates.yml'] = textwrap.dedent("""\
+    content: 'content'
+    variable: '{{woot}}'
+    """)
+
+files['/fixtures/extend_parent.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'extend_child.yml'
+
+    list: ['4']
+    was_list:
+      replaced: true
+    obj:
+      level: 2
+      from_parent: true
+      deeper:
+        list: ['bar']
+    """)
+
+
+class TemplatesTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mocked_open = mozunit.MockedOpen(files)
+        self.mocked_open.__enter__()
+        self.subject = Templates('/fixtures')
+
+    def tearDown(self):
+        self.mocked_open.__exit__(None, None, None)
+
+    def test_invalid_path(self):
+        with self.assertRaisesRegexp(TemplatesException, 'must be a directory'):
+            Templates('/zomg/not/a/dir')
+
+    def test_no_templates(self):
+        content = self.subject.load('simple.yml', {})
+        self.assertEquals(content, {
+            'is_simple': True
+        })
+
+    def test_with_templates(self):
+        content = self.subject.load('templates.yml', {
+            'woot': 'bar'
+        })
+
+        self.assertEquals(content, {
+            'content': 'content',
+            'variable': 'bar'
+        })
+
+    def test_inheritance(self):
+        '''
+        The simple single pass inheritance case.
+        '''
+        content = self.subject.load('inherit.yml', {})
+        self.assertEqual(content, {
+            'content': 'content',
+            'variable': 'inherit'
+        })
+
+    def test_inheritance_implicat_pass(self):
+        '''
+        Implicitly pass parameters from the child to the ancestor.
+        '''
+        content = self.subject.load('inherit_pass.yml', {
+            'a': 'overriden'
+        })
+
+        self.assertEqual(content, { 'values': ['overriden', 'b', 'c'] });
+
+
+    def test_inheritance_circular(self):
+        '''
+        Circular reference handling.
+        '''
+        with self.assertRaisesRegexp(TemplatesException, 'circular'):
+            self.subject.load('circular.yml', {})
+
+    def test_deep_inheritance(self):
+        content = self.subject.load('deep/4.yml', {
+            'value': 'myvalue'
+        })
+        self.assertEqual(content, { 'variable': 'myvalue' })
+
+    def test_inheritance_with_simple_extensions(self):
+        content = self.subject.load('extend_parent.yml', {})
+        self.assertEquals(content, {
+            'list': ['1', '2', '3', '4'],
+            'obj': {
+                'from_parent': True,
+                'deeper': {
+                    'woot': 'bar',
+                    'list': ['baz', 'bar']
+                },
+                'level': 2,
+            },
+            'was_list': { 'replaced': True }
+        })
+
+
+if __name__ == '__main__':
+    mozunit.main()

taskcluster/taskgraph/util/docker.py

@@ -6,12 +6,12 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
 
 def docker_image(name):
-    ''' Determine the docker image name, including repository and tag, from an
+    '''Determine the docker image name, including repository and tag, from an
-    in-tree docker file'''
+    in-tree docker file.'''
     try:
         with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f:
             registry = f.read().strip()
@@ -23,3 +23,4 @@ def docker_image(name):
         version = f.read().strip()
 
     return '{}/{}:{}'.format(registry, name, version)
+

testing/taskcluster/taskcluster_graph/image_builder.py

@@ -8,7 +8,7 @@ import urllib2
 import taskcluster_graph.transform.routes as routes_transform
 import taskcluster_graph.transform.treeherder as treeherder_transform
 from slugid import nice as slugid
-from taskcluster_graph.templates import Templates
+from taskgraph.util.templates import Templates
 
 TASKCLUSTER_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
 IMAGE_BUILD_TASK = os.path.join(TASKCLUSTER_ROOT, 'tasks', 'image.yml')

testing/taskcluster/tests/fixtures/child_pass.yml

@@ -1,4 +0,0 @@
-values:
-  - {{a}}
-  - {{b}}
-  - {{c}}

testing/taskcluster/tests/fixtures/circular.yml

@@ -1,4 +0,0 @@
-$inherits:
-  from: 'circular_ref.yml'
-  variables:
-    woot: 'inherit'

testing/taskcluster/tests/fixtures/circular_ref.yml

@@ -1,2 +0,0 @@
-$inherits:
-  from: 'circular.yml'

testing/taskcluster/tests/fixtures/deep/1.yml

@@ -1 +0,0 @@
-variable: {{value}}

testing/taskcluster/tests/fixtures/deep/2.yml

@@ -1,3 +0,0 @@
-$inherits:
-  from: deep/1.yml
-

testing/taskcluster/tests/fixtures/deep/3.yml

@@ -1,3 +0,0 @@
-$inherits:
-  from: deep/2.yml
-

testing/taskcluster/tests/fixtures/deep/4.yml

@@ -1,2 +0,0 @@
-$inherits:
-  from: deep/3.yml

testing/taskcluster/tests/fixtures/extend_child.yml

@@ -1,7 +0,0 @@
-list: ['1', '2', '3']
-was_list: ['1']
-obj:
-  level: 1
-  deeper:
-    woot: 'bar'
-    list: ['baz']

testing/taskcluster/tests/fixtures/extend_parent.yml

@@ -1,11 +0,0 @@
-$inherits:
-  from: 'extend_child.yml'
-
-list: ['4']
-was_list:
-  replaced: true
-obj:
-  level: 2
-  from_parent: true
-  deeper:
-    list: ['bar']

testing/taskcluster/tests/fixtures/inherit.yml

@@ -1,4 +0,0 @@
-$inherits:
-  from: 'templates.yml'
-  variables:
-    woot: 'inherit'

testing/taskcluster/tests/fixtures/inherit_pass.yml

@@ -1,6 +0,0 @@
-$inherits:
-  from: 'child_pass.yml'
-  variables:
-    a: 'a'
-    b: 'b'
-    c: 'c'

testing/taskcluster/tests/fixtures/simple.yml

@@ -1 +0,0 @@
-is_simple: true

testing/taskcluster/tests/fixtures/templates.yml

@@ -1,2 +0,0 @@
-content: 'content'
-variable: '{{woot}}'

testing/taskcluster/tests/test_templates.py

@@ -1,88 +0,0 @@
-import os
-
-import unittest
-import mozunit
-from taskcluster_graph.templates import (
-    Templates,
-    TemplatesException
-)
-
-class TemplatesTest(unittest.TestCase):
-
-    def setUp(self):
-        abs_path = os.path.abspath(os.path.dirname(__file__))
-        self.subject = Templates(os.path.join(abs_path, 'fixtures'))
-
-
-    def test_invalid_path(self):
-        with self.assertRaisesRegexp(TemplatesException, 'must be a directory'):
-            Templates('/zomg/not/a/dir')
-
-    def test_no_templates(self):
-        content = self.subject.load('simple.yml', {})
-        self.assertEquals(content, {
-            'is_simple': True
-        })
-
-    def test_with_templates(self):
-        content = self.subject.load('templates.yml', {
-            'woot': 'bar'
-        })
-
-        self.assertEquals(content, {
-            'content': 'content',
-            'variable': 'bar'
-        })
-
-    def test_inheritance(self):
-        '''
-        The simple single pass inheritance case.
-        '''
-        content = self.subject.load('inherit.yml', {})
-        self.assertEqual(content, {
-            'content': 'content',
-            'variable': 'inherit'
-        })
-
-    def test_inheritance_implicat_pass(self):
-        '''
-        Implicitly pass parameters from the child to the ancestor.
-        '''
-        content = self.subject.load('inherit_pass.yml', {
-            'a': 'overriden'
-        })
-
-        self.assertEqual(content, { 'values': ['overriden', 'b', 'c'] });
-
-
-    def test_inheritance_circular(self):
-        '''
-        Circular reference handling.
-        '''
-        with self.assertRaisesRegexp(TemplatesException, 'circular'):
-            self.subject.load('circular.yml', {})
-
-    def test_deep_inheritance(self):
-        content = self.subject.load('deep/4.yml', {
-            'value': 'myvalue'
-        })
-        self.assertEqual(content, { 'variable': 'myvalue' })
-
-    def test_inheritance_with_simple_extensions(self):
-        content = self.subject.load('extend_parent.yml', {})
-        self.assertEquals(content, {
-            'list': ['1', '2', '3', '4'],
-            'obj': {
-                'from_parent': True,
-                'deeper': {
-                    'woot': 'bar',
-                    'list': ['baz', 'bar']
-                },
-                'level': 2,
-            },
-            'was_list': { 'replaced': True }
-        })
-
-
-if __name__ == '__main__':
-    mozunit.main()