From ab6940a96ce0eeed10870b51fb02b5d13da61531 Mon Sep 17 00:00:00 2001 From: Henry Fuller Date: Thu, 30 Jun 2022 20:24:12 -0700 Subject: [PATCH] Patch pip environment with xcode sdk location. (#697) --- docs/BUILD | 1 + docs/pip.md | 7 +- examples/pip_install/BUILD | 1 + examples/pip_install/WORKSPACE | 2 +- examples/pip_install/main.py | 1 - examples/pip_install/requirements.in | 1 + examples/pip_install/requirements.txt | 4 + examples/pip_install/requirements_windows.txt | 101 ++++++++ .../parse_requirements_to_bzl/__init__.py | 2 +- python/pip_install/pip_compile.py | 240 ++++++++++-------- python/pip_install/pip_repository.bzl | 43 +++- python/pip_install/requirements.bzl | 18 +- python/repositories.bzl | 3 + 13 files changed, 308 insertions(+), 116 deletions(-) create mode 100644 examples/pip_install/requirements_windows.txt diff --git a/docs/BUILD b/docs/BUILD index 94a7c8451a..d2958219f0 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -62,6 +62,7 @@ bzl_library( bzl_library( name = "pip_install_bzl", srcs = [ + "//python:bzl", "//python/pip_install:bzl", ], deps = [ diff --git a/docs/pip.md b/docs/pip.md index 732d391c5b..4853e5252d 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -5,8 +5,8 @@ ## compile_pip_requirements 
-compile_pip_requirements(name, extra_args, visibility, requirements_in, requirements_txt, tags,
-                         kwargs)
+compile_pip_requirements(name, extra_args, visibility, requirements_in, requirements_txt,
+                         requirements_linux, requirements_darwin, requirements_windows, tags, kwargs)
Generates targets for managing pip dependencies with pip-compile. @@ -31,6 +31,9 @@ It also generates two targets for running pip-compile: | visibility | passed to both the _test and .update rules | ["//visibility:private"] | | requirements_in | file expressing desired dependencies | None | | requirements_txt | result of "compiling" the requirements.in file | None | +| requirements_linux | File of linux specific resolve output to check validate if requirement.in has changes. | None | +| requirements_darwin | File of darwin specific resolve output to check validate if requirement.in has changes. | None | +| requirements_windows | File of windows specific resolve output to check validate if requirement.in has changes. | None | | tags | tagging attribute common to all build rules, passed to both the _test and .update rules | None | | kwargs | other bazel attributes passed to the "_test" rule | none | diff --git a/examples/pip_install/BUILD b/examples/pip_install/BUILD index ecc083fdec..ad983b2f54 100644 --- a/examples/pip_install/BUILD +++ b/examples/pip_install/BUILD @@ -62,6 +62,7 @@ alias( compile_pip_requirements( name = "requirements", extra_args = ["--allow-unsafe"], + requirements_windows = ":requirements_windows.txt", ) # Test the use of all pip_install utilities in a single py_test diff --git a/examples/pip_install/WORKSPACE b/examples/pip_install/WORKSPACE index 1fabf9428e..0b33a2b390 100644 --- a/examples/pip_install/WORKSPACE +++ b/examples/pip_install/WORKSPACE @@ -50,7 +50,7 @@ pip_install( # (Optional) You can set an environment in the pip process to control its # behavior. Note that pip is run in "isolated" mode so no PIP__ # style env vars are read, but env vars that control requests and urllib3 - # can be passed + # can be passed. #environment = {"HTTP_PROXY": "http://my.proxy.fun/"}, # Uses the default repository name "pip" diff --git a/examples/pip_install/main.py b/examples/pip_install/main.py index b65ad0e5ea..4440cdeb2e 100644 --- a/examples/pip_install/main.py +++ b/examples/pip_install/main.py @@ -1,6 +1,5 @@ import boto3 - def the_dir(): return dir(boto3) diff --git a/examples/pip_install/requirements.in b/examples/pip_install/requirements.in index 2351369d18..11ede3c44a 100644 --- a/examples/pip_install/requirements.in +++ b/examples/pip_install/requirements.in @@ -1,3 +1,4 @@ boto3~=1.14.51 s3cmd~=2.1.0 yamllint~=1.26.3 +tree-sitter==0.20.0 ; sys_platform != "win32" diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index 26de1adaac..db76801f39 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -86,6 +86,10 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via python-dateutil +tree-sitter==0.20.0 ; sys_platform != "win32" \ + --hash=sha256:1940f64be1e8c9c3c0e34a2258f1e4c324207534d5b1eefc5ab2960a9d98f668 \ + --hash=sha256:51a609a7c1bd9d9e75d92ee128c12c7852ae70a482900fbbccf3d13a79e0378c + # via -r requirements.in urllib3==1.25.11 \ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e diff --git a/examples/pip_install/requirements_windows.txt b/examples/pip_install/requirements_windows.txt new file mode 100644 index 0000000000..26de1adaac --- /dev/null +++ b/examples/pip_install/requirements_windows.txt @@ -0,0 +1,101 @@ +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# bazel run //:requirements.update +# +boto3==1.14.51 \ + --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \ + --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71 + # via -r requirements.in +botocore==1.17.63 \ + --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \ + --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 + # via + # boto3 + # s3transfer +docutils==0.15.2 \ + --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ + --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ + --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 + # via botocore +jmespath==0.10.0 \ + --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \ + --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f + # via + # boto3 + # botocore +pathspec==0.9.0 \ + --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ + --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 + # via yamllint +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # botocore + # s3cmd +python-magic==0.4.24 \ + --hash=sha256:4fec8ee805fea30c07afccd1592c0f17977089895bdfaae5fec870a84e997626 \ + --hash=sha256:de800df9fb50f8ec5974761054a708af6e4246b03b4bdaee993f948947b0ebcf + # via s3cmd +pyyaml==6.0 \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via yamllint +s3cmd==2.1.0 \ + --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ + --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 + # via -r requirements.in +s3transfer==0.3.7 \ + --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ + --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 + # via boto3 +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via python-dateutil +urllib3==1.25.11 \ + --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ + --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e + # via botocore +yamllint==1.26.3 \ + --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==59.6.0 \ + --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ + --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e + # via yamllint diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index 83526a7a09..07ee92d9d0 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py @@ -94,7 +94,7 @@ def generate_parsed_requirements_contents( repository rule, which will represent the individual requirements. Generates a requirements.bzl file containing a macro (install_deps()) which instantiates - a repository rule for each requirment in the lock file. + a repository rule for each requirement in the lock file. """ install_req_and_lines = parse_install_requirements( requirements_lock, whl_library_args["extra_pip_args"] diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py index e9c5cdf05c..aeb36de049 100644 --- a/python/pip_install/pip_compile.py +++ b/python/pip_install/pip_compile.py @@ -6,116 +6,146 @@ from piptools.scripts.compile import cli -if len(sys.argv) < 4: - print( - "Expected at least two arguments: requirements_in requirements_out", - file=sys.stderr, - ) - sys.exit(1) - -requirements_in = os.path.relpath(sys.argv.pop(1)) -requirements_txt = os.path.relpath(sys.argv.pop(1)) -parts = requirements_in.split(os.path.sep, 2) -if parts[0] == "external": - requirements_in = parts[2] - requirements_txt = requirements_txt if "BUILD_WORKSPACE_DIRECTORY" in os.environ else os.path.join("..", "..", requirements_txt) - os.chdir(os.path.join(parts[0], parts[1])) -update_target_label = sys.argv.pop(1) - -# Before loading click, set the locale for its parser. -# If it leaks through to the system setting, it may fail: -# RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII -# as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for -# mitigation steps. -os.environ["LC_ALL"] = "C.UTF-8" -os.environ["LANG"] = "C.UTF-8" - -UPDATE = True -# Detect if we are running under `bazel test` -if "TEST_TMPDIR" in os.environ: - UPDATE = False - # pip-compile wants the cache files to be writeable, but if we point - # to the real user cache, Bazel sandboxing makes the file read-only - # and we fail. - # In theory this makes the test more hermetic as well. - sys.argv.append("--cache-dir") - sys.argv.append(os.environ["TEST_TMPDIR"]) - # Make a copy for pip-compile to read and mutate - requirements_out = os.path.join( - os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" - ) - copyfile(requirements_txt, requirements_out) - -elif "BUILD_WORKSPACE_DIRECTORY" in os.environ: - # This value, populated when running under `bazel run`, is a path to the - # "root of the workspace where the build was run." - # This matches up with the values passed in via the macro using the 'rootpath' Make variable, - # which for source files provides a path "relative to your workspace root." - # - # Changing to the WORKSPACE root avoids 'file not found' errors when the `.update` target is run - # from different directories within the WORKSPACE. - requirements_txt = os.path.join(os.environ["BUILD_WORKSPACE_DIRECTORY"], requirements_txt) -else: - err_msg = ( - "Expected to find BUILD_WORKSPACE_DIRECTORY (running under `bazel run`) or " - "TEST_TMPDIR (running under `bazel test`) in environment." - ) - print( - err_msg, - file=sys.stderr, + +def _select_golden_requirements_file( + requirements_txt, requirements_linux, requirements_darwin, requirements_windows +): + """Switch the golden requirements file, used to validate if updates are needed, + to a specified platform specific one. Fallback on the platform independent one. + """ + + plat = sys.platform + if plat == "linux" and requirements_linux is not None: + return requirements_linux + elif plat == "darwin" and requirements_darwin is not None: + return requirements_darwin + elif plat == "win32" and requirements_windows is not None: + return requirements_windows + else: + return requirements_txt + + +if __name__ == "__main__": + if len(sys.argv) < 4: + print( + "Expected at least two arguments: requirements_in requirements_out", + file=sys.stderr, + ) + sys.exit(1) + + parse_str_none = lambda s: None if s == "None" else s + + requirements_in = os.path.relpath(sys.argv.pop(1)) + requirements_txt = os.path.relpath(sys.argv.pop(1)) + requirements_linux = parse_str_none(sys.argv.pop(1)) + requirements_darwin = parse_str_none(sys.argv.pop(1)) + requirements_windows = parse_str_none(sys.argv.pop(1)) + parts = requirements_in.split(os.path.sep, 2) + if parts[0] == "external": + requirements_in = parts[2] + requirements_txt = requirements_txt if "BUILD_WORKSPACE_DIRECTORY" in os.environ else os.path.join("..", "..", requirements_txt) + os.chdir(os.path.join(parts[0], parts[1])) + update_target_label = sys.argv.pop(1) + + # Before loading click, set the locale for its parser. + # If it leaks through to the system setting, it may fail: + # RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII + # as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for + # mitigation steps. + os.environ["LC_ALL"] = "C.UTF-8" + os.environ["LANG"] = "C.UTF-8" + + UPDATE = True + # Detect if we are running under `bazel test` + if "TEST_TMPDIR" in os.environ: + UPDATE = False + # pip-compile wants the cache files to be writeable, but if we point + # to the real user cache, Bazel sandboxing makes the file read-only + # and we fail. + # In theory this makes the test more hermetic as well. + sys.argv.append("--cache-dir") + sys.argv.append(os.environ["TEST_TMPDIR"]) + # Make a copy for pip-compile to read and mutate + requirements_out = os.path.join( + os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" + ) + copyfile(requirements_txt, requirements_out) + + elif "BUILD_WORKSPACE_DIRECTORY" in os.environ: + # This value, populated when running under `bazel run`, is a path to the + # "root of the workspace where the build was run." + # This matches up with the values passed in via the macro using the 'rootpath' Make variable, + # which for source files provides a path "relative to your workspace root." + # + # Changing to the WORKSPACE root avoids 'file not found' errors when the `.update` target is run + # from different directories within the WORKSPACE. + requirements_txt = os.path.join(os.environ["BUILD_WORKSPACE_DIRECTORY"], requirements_txt) + else: + err_msg = ( + "Expected to find BUILD_WORKSPACE_DIRECTORY (running under `bazel run`) or " + "TEST_TMPDIR (running under `bazel test`) in environment." + ) + print( + err_msg, + file=sys.stderr, + ) + sys.exit(1) + + update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( + update_target_label, ) - sys.exit(1) - -update_target_pkg = "/".join(requirements_in.split("/")[:-1]) -# $(rootpath) in the workspace root gives ./requirements.in -if update_target_pkg == ".": - update_target_pkg = "" -update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( - update_target_label, -) - -os.environ["CUSTOM_COMPILE_COMMAND"] = update_command -os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull - -sys.argv.append("--generate-hashes") -sys.argv.append("--output-file") -sys.argv.append(requirements_txt if UPDATE else requirements_out) -sys.argv.append(requirements_in) - -if UPDATE: - print("Updating " + requirements_txt) - cli() -else: - # cli will exit(0) on success - try: - print("Checking " + requirements_txt) + + os.environ["CUSTOM_COMPILE_COMMAND"] = update_command + os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull + + sys.argv.append("--generate-hashes") + sys.argv.append("--output-file") + sys.argv.append(requirements_txt if UPDATE else requirements_out) + sys.argv.append(requirements_in) + + if UPDATE: + print("Updating " + requirements_txt) cli() - print("cli() should exit", file=sys.stderr) - sys.exit(1) - except SystemExit as e: - if e.code == 2: - print( - "pip-compile exited with code 2. This means that pip-compile found " - "incompatible requirements or could not find a version that matches " - f"the install requirement in {requirements_in}.", - file=sys.stderr, - ) + else: + # cli will exit(0) on success + try: + print("Checking " + requirements_txt) + cli() + print("cli() should exit", file=sys.stderr) sys.exit(1) - elif e.code == 0: - golden = open(requirements_txt).readlines() - out = open(requirements_out).readlines() - if golden != out: - import difflib + except SystemExit as e: + if e.code == 2: + print( + "pip-compile exited with code 2. This means that pip-compile found " + "incompatible requirements or could not find a version that matches " + f"the install requirement in {requirements_in}.", + file=sys.stderr, + ) + sys.exit(1) + elif e.code == 0: + golden_filename = _select_golden_requirements_file( + requirements_txt, + requirements_darwin, + requirements_linux, + requirements_windows, + ) + golden = open(golden_filename).readlines() + out = open(requirements_out).readlines() + if golden != out: + import difflib - print("".join(difflib.unified_diff(golden, out)), file=sys.stderr) + print("".join(difflib.unified_diff(golden, out)), file=sys.stderr) + print( + "Lock file out of date. Run '" + + update_command + + "' to update.", + file=sys.stderr, + ) + sys.exit(1) + sys.exit(0) + else: print( - "Lock file out of date. Run '" + update_command + "' to update.", + f"pip-compile unexpectedly exited with code {e.code}.", file=sys.stderr, ) sys.exit(1) - sys.exit(0) - else: - print( - f"pip-compile unexpectedly exited with code {e.code}.", file=sys.stderr - ) - sys.exit(1) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index d9888a2990..743d3e3787 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -1,8 +1,13 @@ "" +load("//python:repositories.bzl", "STANDALONE_INTERPRETER_FILENAME") load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") +CPPFLAGS = "CPPFLAGS" + +COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" + def _construct_pypath(rctx): """Helper function to construct a PYTHONPATH. @@ -61,6 +66,29 @@ def _resolve_python_interpreter(rctx): fail("python interpreter `{}` not found in PATH".format(python_interpreter)) return python_interpreter +def _maybe_set_xcode_location_cflags(rctx, environment): + """Patch environment with CPPFLAGS of xcode sdk location. + + Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. + Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg + otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 + """ + if ( + rctx.os.name.lower().startswith("mac os") and + rctx.attr.python_interpreter_target != None and + rctx.path(Label("@{}//:{}".format(rctx.attr.python_interpreter_target.workspace_name, STANDALONE_INTERPRETER_FILENAME))) and + not environment.get(CPPFLAGS) + ): + xcode_sdk_location = rctx.execute(["xcode-select", "--print-path"]) + if xcode_sdk_location.return_code == 0: + xcode_root = xcode_sdk_location.stdout.strip() + if COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): + # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer + # so we need to change the path to to the macos specific tools which are in a different relative + # path than xcode installed command line tools. + xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root) + environment[CPPFLAGS] = "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root) + def _parse_optional_attrs(rctx, args): """Helper function to parse common attributes of pip_repository and whl_library repository rules. @@ -112,6 +140,17 @@ def _parse_optional_attrs(rctx, args): return args +def _create_repository_execution_environment(rctx): + """Create a environment dictionary for processes we spawn with rctx.execute. + + Args: + rctx: The repository context. + Returns: Dictionary of envrionment variable suitable to pass to rctx.execute. + """ + env = {"PYTHONPATH": _construct_pypath(rctx)} + _maybe_set_xcode_location_cflags(rctx, env) + return env + _BUILD_FILE_CONTENTS = """\ package(default_visibility = ["//visibility:public"]) @@ -186,7 +225,7 @@ def _pip_repository_impl(rctx): result = rctx.execute( args, # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = {"PYTHONPATH": _construct_pypath(rctx)}, + environment = _create_repository_execution_environment(rctx), timeout = rctx.attr.timeout, quiet = rctx.attr.quiet, ) @@ -390,7 +429,7 @@ def _whl_library_impl(rctx): result = rctx.execute( args, # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = {"PYTHONPATH": _construct_pypath(rctx)}, + environment = _create_repository_execution_environment(rctx), quiet = rctx.attr.quiet, timeout = rctx.attr.timeout, ) diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 0035e7c8a2..0c661339b1 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -9,6 +9,9 @@ def compile_pip_requirements( visibility = ["//visibility:private"], requirements_in = None, requirements_txt = None, + requirements_linux = None, + requirements_darwin = None, + requirements_windows = None, tags = None, **kwargs): """Generates targets for managing pip dependencies with pip-compile. @@ -28,6 +31,9 @@ def compile_pip_requirements( visibility: passed to both the _test and .update rules requirements_in: file expressing desired dependencies requirements_txt: result of "compiling" the requirements.in file + requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. + requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. + requirements_windows: File of windows specific resolve output to check validate if requirement.in has changes. tags: tagging attribute common to all build rules, passed to both the _test and .update rules **kwargs: other bazel attributes passed to the "_test" rule """ @@ -43,17 +49,21 @@ def compile_pip_requirements( visibility = visibility, ) - data = [name, requirements_in, requirements_txt] + data = [name, requirements_in, requirements_txt] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] # Use the Label constructor so this is expanded in the context of the file # where it appears, which is to say, in @rules_python pip_compile = Label("//python/pip_install:pip_compile.py") - loc = "$(rootpath %s)" + loc = "$(rootpath {})" args = [ - loc % requirements_in, - loc % requirements_txt, + loc.format(requirements_in), + loc.format(requirements_txt), + # String None is a placeholder for argv ordering. + loc.format(requirements_linux) if requirements_linux else "None", + loc.format(requirements_darwin) if requirements_darwin else "None", + loc.format(requirements_windows) if requirements_windows else "None", "//%s:%s.update" % (native.package_name(), name), ] + extra_args diff --git a/python/repositories.bzl b/python/repositories.bzl index dc3ca06709..1441432547 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -35,6 +35,8 @@ def py_repositories(): # Remaining content of the file is only used to support toolchains. ######## +STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER" + def _python_repository_impl(rctx): if rctx.attr.distutils and rctx.attr.distutils_content: fail("Only one of (distutils, distutils_content) should be set.") @@ -187,6 +189,7 @@ py_runtime_pair( python_path = python_bin, python_version = python_short_version, ) + rctx.file(STANDALONE_INTERPRETER_FILENAME, "# File intentionally left blank. Indicates that this is an interpreter repo created by rules_python.") rctx.file("BUILD.bazel", build_content) return {