Merge branch 'master' into release

.github/workflows/taskcluster.yml

@@ -12,6 +12,6 @@ jobs:
       - run: sudo -H python2.7 -m pip install pip==20.3.4 wheel==0.37.0 --upgrade
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
-      - run: sudo -H python3 -m pip install pip==20.3.4 wheel==0.37.0 --upgrade
+          python-version: '3.9.14'
+      - run: python3 -m pip install pip==20.3.4 wheel==0.37.0 --upgrade
       - run: CI/start-worker.sh worker-$TC_WORKER_TYPE $TC_WORKER_TYPE

CI/decision.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import hashlib
 import json
 import os
@@ -412,8 +416,8 @@ def main():
                 merge_coverage,
                 [
                     'cd repo',
-                    'codecov --required --name "taskcluster" --commit {}'
-                    ' --branch {}'.format(TC_COMMIT, TC_BRANCH),
+                    'codecov -Z --name "taskcluster" -C {} -B {}'
+                    .format(TC_COMMIT, TC_BRANCH),
                 ],
             )),
         )

CI/docker.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import hashlib
 import json
 
@@ -166,7 +170,11 @@ def sources_list(snapshot, sections):
             ])),
             'apt-get clean',
             'ln -s /usr/bin/python3-coverage /usr/local/bin/coverage',
-            'python3 -m pip install codecov==2.1.12',
+            'curl -o /usr/local/bin/codecov -sL {}'.format(
+                'https://github.com/codecov/uploader/releases/download'
+                '/v0.1.0_9779/codecov-linux'
+            ),
+            'chmod +x /usr/local/bin/codecov',
             'curl -sL {} | tar -C /usr/local/bin -jxf -'.format(
                 'https://github.com/mozilla/grcov/releases/download/v0.8.7'
                 '/grcov-x86_64-unknown-linux-gnu.tar.bz2'

CI/filter.py

@@ -0,0 +1,60 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+
+class DataCommand:
+    def __init__(self, data):
+        self.data = data
+
+    def write_to(self, out):
+        out.write(b"data %d\n" % len(self.data))
+        out.write(self.data)
+
+
+def iter_commands(input=sys.stdin.buffer):
+    for line in input:
+        if line.startswith(b"data "):
+            _, length = line.split()
+            length = int(length)
+            data = input.read(length)
+            assert len(data) == length
+            yield DataCommand(data)
+        else:
+            yield line
+
+
+def write_command(command, out=sys.stdout.buffer):
+    if isinstance(command, DataCommand):
+        command.write_to(out)
+    else:
+        out.write(command)
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    for arg in args:
+        if arg not in ["--commits", "--roots"]:
+            print(f"Unsupported options: {args}")
+            sys.exit(1)
+
+    commands = iter_commands()
+    for command in commands:
+        if isinstance(command, DataCommand):
+            if "--commits" in args:
+                command.data += b"\n"
+            if "--roots" in args:
+                write_command(command)
+                while True:
+                    command = next(commands)
+                    # No "from" command, so this is a root, remove all the
+                    # files from it.
+                    if not command.startswith((b"deleteall", b"M ")):
+                        break
+
+        elif command.startswith((b"author  <", b"committer  <")):
+            cmd, email = command.split(b"<", 1)
+            command = cmd[:-1] + email.split(b"@", 1)[0] + b" <" + email
+        write_command(command)

CI/hg-serve-exec.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 from __future__ import print_function
 import base64
 import os

CI/msys.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import hashlib
 import re
 

CI/osx.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import hashlib
 
 from tasks import (

CI/package.mk

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 TOPDIR = $(abspath $(or $(dir $(firstword $(MAKEFILE_LIST))),$(CURDIR))/..)
 
 ifeq (a,$(firstword a$(subst /, ,$(abspath .))))

CI/start-worker.sh

@@ -1,4 +1,7 @@
 #!/bin/sh
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 where=$(dirname $0)
 clientId=$1

CI/tasks.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import base64
 import datetime
 import json
@@ -354,7 +358,7 @@ def file_format(url):
             elif k == 'dockerSave':
                 features = task['payload'].setdefault('features', {})
                 features[k] = bool(v)
-                artifact_paths.append('dockerImage.tar')
+                artifact_paths.append('public/dockerImage.tar')
             else:
                 raise Exception("Don't know how to handle {}".format(k))
         task['dependencies'] = sorted(dependencies)

CI/tests.mk

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 TOPDIR = $(abspath $(or $(dir $(firstword $(MAKEFILE_LIST))),$(CURDIR))/..)
 
 ifeq (a,$(firstword a$(subst /, ,$(abspath .))))
@@ -232,12 +236,11 @@ hg.graft.base.git hg.graft2.base.git: hg.upgraded.git hg.pure.hg
 	$(GIT) init $@
 	$(GIT) -C $@ remote add origin hg::$(PATH_URL)/$(word 2,$^)
 	$(GIT) -C $< push $(CURDIR)/$@ refs/remotes/*:refs/remotes/*
-	$(GIT) -C $@ checkout $$($(GIT) -C $< rev-parse HEAD)
 
 hg.graft.git: hg.graft.base.git hg.upgraded.git
 	cp -r $< $@
 	$(GIT) -C $@ cinnabar rollback 0000000000000000000000000000000000000000
-	$(GIT) -c user.email=foo@bar -C $@ filter-branch --msg-filter 'cat ; echo' --original original -- --all
+	$(GIT) -C $@ fast-export --no-data --all | python3 $(TOPDIR)/CI/filter.py --commits | git -c core.ignorecase=false -C $@ fast-import --force
 	$(GIT) -C $@ -c cinnabar.graft=true remote update
 	$(call COMPARE_REFS, $(word 2,$^), $@, XARGS_GIT2HG)
 	$(GIT) -C $@ cinnabar fsck --full
@@ -250,10 +253,10 @@ hg.graft2.git: hg.graft.git hg.pure.hg hg.graft2.base.git
 	$(call COMPARE_REFS, $<, $@)
 	$(GIT) -C $@ cinnabar fsck --full
 
-hg.graft.replace.git: hg.graft.git hg.upgraded.git
+hg.graft.replace.git: hg.graft.base.git hg.upgraded.git
 	cp -r $< $@
 	$(GIT) -C $@ cinnabar rollback 0000000000000000000000000000000000000000
-	$(GIT) -c user.email=foo@bar -C $@ filter-branch --index-filter 'test $$GIT_COMMIT = '$$($(call GET_ROOTS,$@,--remotes))' && git rm -r --cached -- \* || true' --original original -- --all
+	$(GIT) -C $@ fast-export --no-data --full-tree --all | python3 $(TOPDIR)/CI/filter.py --commits --roots | git -c core.ignorecase=false -C $@ fast-import --force
 	$(GIT) -C $@ -c cinnabar.graft=true remote update
 	$(call COMPARE_REFS, $(word 2,$^), $@, XARGS_GIT2HG)
 	$(call COMPARE_COMMANDS,$(call GET_ROOTS,$(word 2,$^),--remotes),$(call GET_ROOTS,$@,--glob=refs/cinnabar/replace))

CI/tools.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import hashlib
 import os
 
@@ -12,7 +16,7 @@
 import msys
 
 
-MERCURIAL_VERSION = '6.3.0'
+MERCURIAL_VERSION = '6.4.2'
 GIT_VERSION = '2.40.0'
 
 ALL_MERCURIAL_VERSIONS = (
@@ -22,7 +26,7 @@
     '4.0.2', '4.1.3', '4.2.2', '4.3.3', '4.4.2', '4.5.3', '4.6.2',
     '4.7.2', '4.8.2', '4.9.1', '5.0.2', '5.1.2', '5.2.2', '5.3.2',
     '5.4.2', '5.5.2', '5.6.1', '5.7.1', '5.8.1', '5.9.3', '6.0.3',
-    '6.1.4', '6.2.3', '6.3.0',
+    '6.1.4', '6.2.3', '6.3.3', '6.4.2',
 )
 
 SOME_MERCURIAL_VERSIONS = (

CI/util.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import os
 import subprocess
 

CI/variables.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import json
 import os