Bug 1762642: Add libopus to mach vendor r=kinetik,jewilde

This makes some ./mach vendor improvements needed:
 - Adds a replace-in-file-regex action
 - Allows you to skip copying the temporary extract directory
   so that a script can do it
 - Removes 'commit' from the look-in-moz.yaml-for-release match

Differential Revision: https://phabricator.services.mozilla.com/D142720

media/libopus/README_MOZILLA

@@ -8,4 +8,4 @@ files after the copy step.
 
 The upstream repository is https://git.xiph.org/opus.git
 
-The git tag/revision used was 2654707e86cc94413998976d179b2ab4a2aa3114.
+Refer to moz.yaml for the current in-tree revision.

media/libopus/gen-sources.py

@@ -64,7 +64,7 @@ def generate_sources_mozbuild(path):
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
-        print "Usage: %s /path/to/opus" % (sys.argv[0])
+        print("Usage: %s /path/to/opus" % (sys.argv[0]))
         sys.exit(1)
 
     generate_sources_mozbuild(sys.argv[1])

media/libopus/moz.yaml

@@ -0,0 +1,67 @@
+# Version of this schema
+schema: 1
+
+bugzilla:
+  # Bugzilla product and component for this directory and subdirectories
+  product: Core
+  component: "Audio/Video: Playback"
+
+# Document the source of externally hosted code
+origin:
+
+  # Short name of the package/library
+  name: opus
+
+  description: opus, an open, royalty-free, highly versatile audio codec
+
+  # Full URL for the package's homepage/etc
+  # Usually different from repository url
+  url: https://opus-codec.org/
+
+  # Human-readable identifier for this version/release
+  # Generally "version NNN", "tag SSS", "bookmark SSS"
+  release: 2654707e86cc94413998976d179b2ab4a2aa3114 (2022-04-01T14:32:38.000-04:00).
+
+  # Revision to pull in
+  # Must be a long or short commit SHA (long preferred)
+  revision: 2654707e86cc94413998976d179b2ab4a2aa3114
+
+  # The package's license, where possible using the mnemonic from
+  # https://spdx.org/licenses/
+  # Multiple licenses can be specified (as a YAML list)
+  # A "LICENSE" file must exist containing the full license text
+  license: BSD-2-Clause
+
+  license-file: COPYING
+
+updatebot:
+    maintainer-phab: kinetik
+    maintainer-bz: kinetik@flim.org
+    tasks:
+      - type: vendoring
+        enabled: true
+        frequency: every
+
+vendoring:
+  url: https://gitlab.xiph.org/xiph/opus
+  source-hosting: gitlab
+  vendor-directory: media/libopus
+  skip-vendoring-steps: ['move-contents', 'update-moz-build']
+
+  keep:
+    - COPYING
+    - gen-sources.py
+    - "*.patch"
+    - README_MOZILLA
+    - sources.mozbuild
+    - update.sh
+
+  update-actions:
+    - action: run-script
+      script: 'update.sh'
+      cwd: '{yaml_dir}'
+      args: ['{tmpextractdir}']
+    - action: replace-in-file-regex
+      file: moz.build
+      pattern: 'DEFINES\["OPUS_VERSION"\] = "(.+)"'
+      with: 'DEFINES["OPUS_VERSION"] = "{revision}"'

media/libopus/update.sh

@@ -61,18 +61,9 @@ if test -d $1/.git; then
 else
   version="UNKNOWN"
 fi
-echo "copied from revision ${version}"
-# update README revision
-sed -e "s/^The git tag\/revision used was .*/The git tag\/revision used was ${version}./" \
-    ${TARGET}/README_MOZILLA > ${TARGET}/README_MOZILLA+ && \
-    mv ${TARGET}/README_MOZILLA+ ${TARGET}/README_MOZILLA
-# update compiled-in version string
-sed -e "s/DEFINES\['OPUS_VERSION'\][ \t]*=[ \t]*'\".*\"'/DEFINES['OPUS_VERSION'] = '\"${version}-mozilla\"'/" \
-    ${TARGET}/moz.build > ${TARGET}/moz.build+ && \
-    mv ${TARGET}/moz.build+ ${TARGET}/moz.build
 
-python gen-sources.py $1
+python3 gen-sources.py $1
 
 # apply outstanding local patches
-patch -p3 < nonunified.patch
-patch -p3 < nonunified2.patch
+patch -p3 --no-backup-if-mismatch < nonunified.patch
+patch -p3 --no-backup-if-mismatch < nonunified2.patch

python/mozbuild/mozbuild/vendor/moz_yaml.py

@@ -161,10 +161,14 @@ vendoring:
   # The valid steps that can be skipped are listed below
   skip-vendoring-steps:
     - fetch
+    - keep
+    - include
     - exclude
-    - update-moz-yaml
+    - move-contents
     - update-actions
     - hg-add
+    - spurious-check
+    - update-moz-yaml
     - update-moz-build
 
   # List of patch files to apply after vendoring. Applied in the order
@@ -234,6 +238,7 @@ vendoring:
   #   - copy-file
   #   - move-dir
   #   - replace-in-file
+  #   - replace-in-file-regex
   #   - delete-path
   #   - run-script
   # Unless otherwise noted, all subfields of action are required.
@@ -242,7 +247,7 @@ vendoring:
   #   from is the source file
   #   to is the destination
   #
-  # If the action is replace-in-file:
+  # If the action is replace-in-file or replace-in-file-regex:
   #   pattern is what in the file to search for. It is an exact strng match.
   #   with is the string to replace it with. Accepts the special keyword
   #     '{revision}' for the commit we are updating to.
@@ -441,6 +446,7 @@ def _schema_1():
                                     "copy-file",
                                     "move-dir",
                                     "replace-in-file",
+                                    "replace-in-file-regex",
                                     "run-script",
                                     "delete-path",
                                 ],
@@ -563,7 +569,7 @@ class UpdateActions(object):
                         "%s action must (only) specify 'from' and 'to' keys"
                         % v["action"]
                     )
-            elif v["action"] == "replace-in-file":
+            elif v["action"] in ["replace-in-file", "replace-in-file-regex"]:
                 if (
                     "pattern" not in v
                     or "with" not in v

python/mozbuild/mozbuild/vendor/vendor_manifest.py

@@ -30,6 +30,10 @@ DEFAULT_KEEP_FILES = ["moz.build", "moz.yaml"]
 DEFAULT_INCLUDE_FILES = []
 
 
+def throwe():
+    raise Exception
+
+
 class VendorManifest(MozbuildObject):
     def should_perform_step(self, step):
         return step not in self.manifest["vendoring"].get("skip-vendoring-steps", [])
@@ -47,6 +51,7 @@ class VendorManifest(MozbuildObject):
     ):
         self.manifest = manifest
         self.yaml_file = yaml_file
+        self._extract_directory = throwe
         self.logInfo = functools.partial(self.log, logging.INFO, "vendor")
         if "vendor-directory" not in self.manifest["vendoring"]:
             self.manifest["vendoring"]["vendor-directory"] = os.path.dirname(
@@ -170,6 +175,9 @@ class VendorManifest(MozbuildObject):
     def get_full_path(self, path, support_cwd=False):
         if support_cwd and path[0:5] == "{cwd}":
             path = path.replace("{cwd}", ".")
+        elif "{tmpextractdir}" in path:
+            # _extract_directory() will throw an exception if it is invalid to use it
+            path = path.replace("{tmpextractdir}", self._extract_directory())
         elif "{yaml_dir}" in path:
             path = path.replace("{yaml_dir}", os.path.dirname(self.yaml_file))
         elif "{vendor_dir}" in path:
@@ -207,7 +215,8 @@ class VendorManifest(MozbuildObject):
         self.logInfo({"url": url}, "Fetching code archive from {url}")
 
         with mozfile.NamedTemporaryFile() as tmptarfile:
-            with tempfile.TemporaryDirectory() as tmpextractdir:
+            tmpextractdir = tempfile.TemporaryDirectory()
+            try:
                 req = requests.get(url, stream=True)
                 for data in req.iter_content(4096):
                     tmptarfile.write(data)
@@ -244,7 +253,7 @@ class VendorManifest(MozbuildObject):
                         mozfile.remove(file)
 
                 self.logInfo({"vd": vendor_dir}, "Unpacking upstream files for {vd}.")
-                tar.extractall(tmpextractdir)
+                tar.extractall(tmpextractdir.name)
 
                 def get_first_dir(p):
                     halves = os.path.split(p)
@@ -258,14 +267,14 @@ class VendorManifest(MozbuildObject):
 
                 # GitLab puts everything down a directory; move it up.
                 if has_prefix:
-                    tardir = mozpath.join(tmpextractdir, one_prefix)
-                    mozfile.copy_contents(tardir, tmpextractdir)
+                    tardir = mozpath.join(tmpextractdir.name, one_prefix)
+                    mozfile.copy_contents(tardir, tmpextractdir.name)
                     mozfile.remove(tardir)
 
                 if self.should_perform_step("include"):
                     self.logInfo({}, "Retaining wanted files from upstream changes.")
                     to_include = self.convert_patterns_to_paths(
-                        tmpextractdir,
+                        tmpextractdir.name,
                         self.manifest["vendoring"].get("include", [])
                         + DEFAULT_INCLUDE_FILES,
                     )
@@ -276,7 +285,7 @@ class VendorManifest(MozbuildObject):
                 if self.should_perform_step("exclude"):
                     self.logInfo({}, "Removing excluded files from upstream changes.")
                     to_exclude = self.convert_patterns_to_paths(
-                        tmpextractdir,
+                        tmpextractdir.name,
                         self.manifest["vendoring"].get("exclude", [])
                         + DEFAULT_EXCLUDE_FILES,
                     )
@@ -305,11 +314,19 @@ class VendorManifest(MozbuildObject):
                                 pass
                     return removed
 
-                while removeEmpty(tmpextractdir):
+                while removeEmpty(tmpextractdir.name):
                     pass
 
                 # Then copy over the directories
-                mozfile.copy_contents(tmpextractdir, vendor_dir)
+                if self.should_perform_step("move-contents"):
+                    self.logInfo({"d": vendor_dir}, "Copying to {d}.")
+                    mozfile.copy_contents(tmpextractdir.name, vendor_dir)
+                else:
+                    self.logInfo({}, "Skipping copying contents into tree.")
+                    self._extract_directory = lambda: tmpextractdir.name
+            except Exception as e:
+                tmpextractdir.cleanup()
+                raise e
 
     def update_yaml(self, revision, timestamp):
         with open(self.yaml_file) as f:
@@ -317,7 +334,7 @@ class VendorManifest(MozbuildObject):
 
         replaced = 0
         replacements = [
-            ["  release: commit", " %s (%s)." % (revision, timestamp)],
+            ["  release:", " %s (%s)." % (revision, timestamp)],
             ["  revision:", " %s" % (revision)],
         ]
 
@@ -417,7 +434,7 @@ class VendorManifest(MozbuildObject):
                 copy_tree(src, dst)
                 shutil.rmtree(src)
 
-            elif update["action"] == "replace-in-file":
+            elif update["action"] in ["replace-in-file", "replace-in-file-regex"]:
                 file = self.get_full_path(update["file"])
 
                 self.logInfo({"file": file}, "action: replace-in-file file: {file}")
@@ -426,7 +443,10 @@ class VendorManifest(MozbuildObject):
                     contents = f.read()
 
                 replacement = update["with"].replace("{revision}", revision)
-                contents = contents.replace(update["pattern"], replacement)
+                if update["action"] == "replace-in-file":
+                    contents = contents.replace(update["pattern"], replacement)
+                else:
+                    contents = re.sub(update["pattern"], replacement, contents)
 
                 with open(file, "w") as f:
                     f.write(contents)
@@ -442,7 +462,15 @@ class VendorManifest(MozbuildObject):
                 for a in update.get("args", []):
                     if a == "{revision}":
                         args.append(revision)
-                    elif any(s in a for s in ["{cwd}", "{vendor_dir}", "{yaml_dir}"]):
+                    elif any(
+                        s in a
+                        for s in [
+                            "{cwd}",
+                            "{vendor_dir}",
+                            "{yaml_dir}",
+                            "{tmpextractdir}",
+                        ]
+                    ):
                         args.append(self.get_full_path(a, support_cwd=True))
                     else:
                         args.append(a)