-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Lib] Add dependency and output path retrieval library functions
- Loading branch information
Showing
8 changed files
with
124 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
""" | ||
This module is Conductor's user library. It contains various utilities that | ||
can be useful in Python scripts that run as Conductor tasks. | ||
""" | ||
|
||
# Path-related utilities. | ||
from .path import * |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
import os | ||
import pathlib | ||
from typing import List | ||
|
||
from conductor.config import ( | ||
DEPS_ENV_VARIABLE_NAME, | ||
DEPS_ENV_PATH_SEPARATOR, | ||
OUTPUT_ENV_VARIABLE_NAME, | ||
) | ||
|
||
|
||
def get_deps_paths() -> List[pathlib.Path]: | ||
""" | ||
Returns a list of the output paths of this task's dependencies. | ||
""" | ||
if DEPS_ENV_VARIABLE_NAME not in os.environ: | ||
raise RuntimeError( | ||
"The {} environment variable was not set. Make sure your code is " | ||
"being executed by Conductor.".format(DEPS_ENV_VARIABLE_NAME) | ||
) | ||
return list( | ||
map( | ||
pathlib.Path, | ||
os.environ[DEPS_ENV_VARIABLE_NAME].split(DEPS_ENV_PATH_SEPARATOR), | ||
) | ||
) | ||
|
||
|
||
def get_output_path() -> pathlib.Path: | ||
""" | ||
Returns the path where this task's outputs should be stored. | ||
""" | ||
if OUTPUT_ENV_VARIABLE_NAME not in os.environ: | ||
raise RuntimeError( | ||
"The {} environment variable was not set. Make sure your code is " | ||
"being executed by Conductor.".format(OUTPUT_ENV_VARIABLE_NAME) | ||
) | ||
return pathlib.Path(os.environ[OUTPUT_ENV_VARIABLE_NAME]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
run_command( | ||
name="one", | ||
run="exit 0", | ||
) | ||
|
||
run_command( | ||
name="two", | ||
run="exit 0", | ||
) | ||
|
||
run_command( | ||
name="deps", | ||
run="python3 deps.py", | ||
deps=[ | ||
":one", | ||
":two", | ||
], | ||
) | ||
|
||
run_command( | ||
name="output_path", | ||
run="python3 output_path.py", | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
import conductor.lib as cond | ||
from conductor.config import TASK_OUTPUT_DIR_SUFFIX | ||
|
||
|
||
def main(): | ||
expected_deps = {"one" + TASK_OUTPUT_DIR_SUFFIX, "two" + TASK_OUTPUT_DIR_SUFFIX} | ||
deps = cond.get_deps_paths() | ||
assert len(deps) > 0 | ||
for dep_path in deps: | ||
assert dep_path.name in expected_deps | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
import conductor.lib as cond | ||
from conductor.config import TASK_OUTPUT_DIR_SUFFIX | ||
|
||
|
||
def main(): | ||
output_dir = cond.get_output_path() | ||
assert output_dir.exists() | ||
assert output_dir.name == "output_path" + TASK_OUTPUT_DIR_SUFFIX | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import pathlib | ||
import pytest | ||
|
||
import conductor.lib as condlib | ||
from .conductor_runner import ConductorRunner, FIXTURE_TEMPLATES | ||
|
||
|
||
def test_get_output_path_non_cond(): | ||
with pytest.raises(RuntimeError): | ||
_ = condlib.get_output_path() | ||
|
||
|
||
def test_get_deps_paths_non_cond(): | ||
with pytest.raises(RuntimeError): | ||
_ = condlib.get_deps_paths() | ||
|
||
|
||
def test_get_deps_paths(tmp_path: pathlib.Path): | ||
cond = ConductorRunner.from_template(tmp_path, FIXTURE_TEMPLATES["lib-test"]) | ||
result = cond.run("//path:deps") | ||
# The deps.py script in lib-test/path/ does the correctness assertions. | ||
assert result.returncode == 0 | ||
|
||
|
||
def test_get_output_path(tmp_path: pathlib.Path): | ||
cond = ConductorRunner.from_template(tmp_path, FIXTURE_TEMPLATES["lib-test"]) | ||
result = cond.run("//path:output_path") | ||
# The output_path.py script in lib-test/path/ does the correctness assertions. | ||
assert result.returncode == 0 |