From c72c7bcf4e0899c275042328e8233e3124ccae86 Mon Sep 17 00:00:00 2001 From: yuvalk Date: Tue, 11 Apr 2023 19:07:34 +0300 Subject: [PATCH] feat: Support specifying multiple download URLs in tool_versions. (#1145) The interface of `repository_ctx.download` and `repository_ctx.download_and_extract` supports string lists as well as strings as the value of the `url` argument. This is the ultimate destination of the `url` attribute in the `tool_versions` dictionary, so it makes sense for it to support lists as well. It is often useful to provide multiple download URLs, e.g. when vendoring deps through a mirror (to guard against issues like [git archive checksums changing](https://github.blog/changelog/2023-01-30-git-archive-checksums-may-change/) while still keeping the canonical download URL) or in an airgapped setting (to support internal URLs alongside external URLs). This is also pretty common around Bazel repository rules that download things, e.g. [http_archive](https://bazel.build/rules/lib/repo/http#http_archive-urls), so it can be expected to work with `tool_versions` too. --- python/repositories.bzl | 14 +++++++++----- python/versions.bzl | 36 ++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index f676610ae2..2429d7e026 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -99,12 +99,14 @@ def is_standalone_interpreter(rctx, python_interpreter_target): def _python_repository_impl(rctx): if rctx.attr.distutils and rctx.attr.distutils_content: fail("Only one of (distutils, distutils_content) should be set.") + if bool(rctx.attr.url) == bool(rctx.attr.urls): + fail("Exactly one of (url, urls) must be set.") platform = rctx.attr.platform python_version = rctx.attr.python_version python_short_version = python_version.rpartition(".")[0] release_filename = rctx.attr.release_filename - url = rctx.attr.url + url = rctx.attr.urls or [rctx.attr.url] if release_filename.endswith(".zst"): rctx.download( @@ -428,8 +430,10 @@ For more information see the official bazel docs doc = "A directory prefix to strip from the extracted files.", ), "url": attr.string( - doc = "The URL of the interpreter to download", - mandatory = True, + doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", + ), + "urls": attr.string_list( + doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", ), "zstd_sha256": attr.string( default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", @@ -506,7 +510,7 @@ def python_register_toolchains( if not sha256: continue - (release_filename, url, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions) + (release_filename, urls, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions) # allow passing in a tool version coverage_tool = None @@ -536,7 +540,7 @@ def python_register_toolchains( platform = platform, python_version = python_version, release_filename = release_filename, - url = url, + urls = urls, distutils = distutils, distutils_content = distutils_content, strip_prefix = strip_prefix, diff --git a/python/versions.bzl b/python/versions.bzl index 4feeeae58c..662f89d04b 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -41,6 +41,8 @@ DEFAULT_RELEASE_BASE_URL = "https://github.com/indygreg/python-build-standalone/ # "strip_prefix": "python", # }, # +# It is possible to provide lists in "url". +# # buildifier: disable=unsorted-dict-items TOOL_VERSIONS = { "3.8.10": { @@ -281,19 +283,28 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U if type(url) == type({}): url = url[platform] + if type(url) != type([]): + url = [url] + strip_prefix = tool_versions[python_version].get("strip_prefix", None) if type(strip_prefix) == type({}): strip_prefix = strip_prefix[platform] - release_filename = url.format( - platform = platform, - python_version = python_version, - build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only", - ) - if "://" in release_filename: # is absolute url? - url = release_filename - else: - url = "/".join([base_url, release_filename]) + release_filename = None + rendered_urls = [] + for u in url: + release_filename = u.format( + platform = platform, + python_version = python_version, + build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only", + ) + if "://" in release_filename: # is absolute url? + rendered_urls.append(release_filename) + else: + rendered_urls.append("/".join([base_url, release_filename])) + + if release_filename == None: + fail("release_filename should be set by now; were any download URLs given?") patches = tool_versions[python_version].get("patches", []) if type(patches) == type({}): @@ -302,7 +313,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U else: patches = [] - return (release_filename, url, strip_prefix, patches) + return (release_filename, rendered_urls, strip_prefix, patches) def print_toolchains_checksums(name): native.genrule( @@ -333,10 +344,11 @@ def _commands_for_version(python_version): "echo \"{python_version}: {platform}: $$(curl --location --fail {release_url_sha256} 2>/dev/null || curl --location --fail {release_url} 2>/dev/null | shasum -a 256 | awk '{{ print $$1 }}')\"".format( python_version = python_version, platform = platform, - release_url = get_release_info(platform, python_version)[1], - release_url_sha256 = get_release_info(platform, python_version)[1] + ".sha256", + release_url = release_url, + release_url_sha256 = release_url + ".sha256", ) for platform in TOOL_VERSIONS[python_version]["sha256"].keys() + for release_url in get_release_info(platform, python_version)[1] ]) def gen_python_config_settings(name = ""):