From 3e2785f7d53dd628e7027fe988ae066fa1be0da1 Mon Sep 17 00:00:00 2001 From: Dustin Ngo Date: Tue, 4 Jun 2024 12:54:46 -0400 Subject: [PATCH] feat: Adds timing info to llm_classify (#3377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add timing info to `ExecutionDetails` payload * Add tests for execution time output * Ruff 🐶 * Include diagnostic columns by default on `llm_classify` * Use variable, not attribute --- .../src/phoenix/evals/classify.py | 23 +- .../src/phoenix/evals/executors.py | 16 +- .../phoenix/evals/functions/test_classify.py | 15 +- .../phoenix/evals/functions/test_executor.py | 6 + .../evaluations_with_error_handling.ipynb | 1147 +++++++---------- 5 files changed, 474 insertions(+), 733 deletions(-) diff --git a/packages/phoenix-evals/src/phoenix/evals/classify.py b/packages/phoenix-evals/src/phoenix/evals/classify.py index 99d974c951..1751094ca7 100644 --- a/packages/phoenix-evals/src/phoenix/evals/classify.py +++ b/packages/phoenix-evals/src/phoenix/evals/classify.py @@ -116,10 +116,6 @@ def llm_classify( include_response (bool, default=False): If True, includes a column named `response` in the output dataframe containing the raw response from the LLM. - include_exceptions (bool, default=False): If True, includes two columns named `exceptions` - and `execution_status` in the output dataframe containing details about execution errors - that may have occurred during the classification. - max_retries (int, optional): The maximum number of times to retry on exceptions. Defaults to 10. @@ -140,7 +136,10 @@ def llm_classify( `explanation` is added to contain the explanation for each label. The dataframe has the same length and index as the input dataframe. The classification label values are from the entries in the rails argument or "NOT_PARSABLE" if the model's output could - not be parsed. + not be parsed. The output dataframe also includes three additional columns in the + output dataframe: `exceptions`, `execution_status`, and `execution_seconds` containing + details about execution errors that may have occurred during the classification as well + as the total runtime of each classification (in seconds). """ concurrency = concurrency or model.default_concurrency # clients need to be reloaded to ensure that async evals work properly @@ -236,6 +235,7 @@ def _run_llm_classification_sync(input_data: pd.Series[Any]) -> ParsedLLMRespons labels, explanations, responses, prompts = zip(*results) all_exceptions = [details.exceptions for details in execution_details] execution_statuses = [details.status for details in execution_details] + execution_times = [details.execution_seconds for details in execution_details] classification_statuses = [] for exceptions, status in zip(all_exceptions, execution_statuses): if exceptions and isinstance(exceptions[-1], PhoenixTemplateMappingError): @@ -249,16 +249,9 @@ def _run_llm_classification_sync(input_data: pd.Series[Any]) -> ParsedLLMRespons **({"explanation": explanations} if provide_explanation else {}), **({"prompt": prompts} if include_prompt else {}), **({"response": responses} if include_response else {}), - **( - {"exceptions": [[repr(exc) for exc in excs] for excs in all_exceptions]} - if include_exceptions - else {} - ), - **( - {"execution_status": [status.value for status in classification_statuses]} - if include_exceptions - else {} - ), + **({"exceptions": [[repr(exc) for exc in excs] for excs in all_exceptions]}), + **({"execution_status": [status.value for status in classification_statuses]}), + **({"execution_seconds": [runtime for runtime in execution_times]}), }, index=dataframe.index, ) diff --git a/packages/phoenix-evals/src/phoenix/evals/executors.py b/packages/phoenix-evals/src/phoenix/evals/executors.py index 992a746165..f090645cc8 100644 --- a/packages/phoenix-evals/src/phoenix/evals/executors.py +++ b/packages/phoenix-evals/src/phoenix/evals/executors.py @@ -4,6 +4,7 @@ import logging import signal import threading +import time from contextlib import contextmanager from enum import Enum from typing import ( @@ -43,6 +44,7 @@ class ExecutionDetails: def __init__(self) -> None: self.exceptions: List[Exception] = [] self.status = ExecutionStatus.DID_NOT_RUN + self.execution_seconds: float = 0 def fail(self) -> None: self.status = ExecutionStatus.FAILED @@ -56,6 +58,9 @@ def complete(self) -> None: def log_exception(self, exc: Exception) -> None: self.exceptions.append(exc) + def log_runtime(self, start_time: float) -> None: + self.execution_seconds += time.time() - start_time + class Executor(Protocol): def run(self, inputs: Sequence[Any]) -> Tuple[List[Any], List[ExecutionDetails]]: ... @@ -154,6 +159,7 @@ async def consumer( index, payload = item try: + task_start_time = time.time() generate_task = asyncio.create_task(self.generate(payload)) termination_event_watcher = asyncio.create_task(termination_event.wait()) done, pending = await asyncio.wait( @@ -165,6 +171,7 @@ async def consumer( if generate_task in done: outputs[index] = generate_task.result() execution_details[index].complete() + execution_details[index].log_runtime(task_start_time) progress_bar.update() elif termination_event.is_set(): # discard the pending task and remaining items in the queue @@ -181,10 +188,12 @@ async def consumer( continue else: tqdm.write("Worker timeout, requeuing") - # task timeouts are requeued at base priority - await queue.put((self.base_priority, item)) + # task timeouts are requeued at the same priority + await queue.put((priority, item)) + execution_details[index].log_runtime(task_start_time) except Exception as exc: execution_details[index].log_exception(exc) + execution_details[index].log_runtime(task_start_time) is_phoenix_exception = isinstance(exc, PhoenixException) if (retry_count := abs(priority)) < self.max_retries and not is_phoenix_exception: tqdm.write( @@ -332,6 +341,7 @@ def run(self, inputs: Sequence[Any]) -> Tuple[List[Any], List[Any]]: progress_bar = tqdm(total=len(inputs), bar_format=self.tqdm_bar_format) for index, input in enumerate(inputs): + task_start_time = time.time() try: for attempt in range(self.max_retries + 1): if self._TERMINATE: @@ -357,6 +367,8 @@ def run(self, inputs: Sequence[Any]) -> Tuple[List[Any], List[Any]]: return outputs, execution_details else: progress_bar.update() + finally: + execution_details[index].log_runtime(task_start_time) return outputs, execution_details diff --git a/packages/phoenix-evals/tests/phoenix/evals/functions/test_classify.py b/packages/phoenix-evals/tests/phoenix/evals/functions/test_classify.py index ec651c22ae..2a7f6484c9 100644 --- a/packages/phoenix-evals/tests/phoenix/evals/functions/test_classify.py +++ b/packages/phoenix-evals/tests/phoenix/evals/functions/test_classify.py @@ -135,7 +135,7 @@ def test_llm_classify( expected_labels = ["relevant", "unrelated", "relevant", NOT_PARSABLE] assert result.iloc[:, 0].tolist() == expected_labels assert_frame_equal( - result, + result[["label"]], pd.DataFrame( data={"label": expected_labels}, ), @@ -224,7 +224,7 @@ def test_llm_classify_with_async( expected_labels = ["relevant", "unrelated", "relevant", NOT_PARSABLE] assert result.iloc[:, 0].tolist() == expected_labels assert_frame_equal( - result, + result[["label"]], pd.DataFrame( data={"label": expected_labels}, ), @@ -258,7 +258,7 @@ def test_llm_classify_with_fn_call( expected_labels = ["relevant", "unrelated", "relevant", NOT_PARSABLE] assert result.iloc[:, 0].tolist() == expected_labels - assert_frame_equal(result, pd.DataFrame(data={"label": expected_labels})) + assert_frame_equal(result[["label"]], pd.DataFrame(data={"label": expected_labels})) @pytest.mark.respx(base_url="https://api.openai.com/v1/chat/completions") @@ -290,7 +290,7 @@ def test_classify_fn_call_no_explain( expected_labels = ["relevant", "unrelated", "relevant", NOT_PARSABLE] assert result.iloc[:, 0].tolist() == expected_labels assert_frame_equal( - result, + result[["label", "explanation"]], pd.DataFrame(data={"label": expected_labels, "explanation": [None, None, None, None]}), ) @@ -328,7 +328,7 @@ def test_classify_fn_call_explain( expected_labels = ["relevant", "unrelated", "relevant", NOT_PARSABLE] assert result.iloc[:, 0].tolist() == expected_labels assert_frame_equal( - result, + result[["label", "explanation"]], pd.DataFrame(data={"label": expected_labels, "explanation": ["0", "1", "2", "3"]}), ) @@ -470,7 +470,6 @@ def test_classify_exits_on_missing_input( template=classification_template, model=model, rails=["relevant", "unrelated"], - include_exceptions=True, max_retries=4, exit_on_error=True, run_sync=True, # run synchronously to ensure ordering @@ -519,7 +518,6 @@ def test_classify_skips_missing_input_with_when_exit_on_error_false( template=classification_template, model=model, rails=["relevant", "unrelated"], - include_exceptions=True, max_retries=4, exit_on_error=False, ) @@ -538,6 +536,9 @@ def test_classify_skips_missing_input_with_when_exit_on_error_false( 1, # one failure due to missing input 5, # first attempt + 4 retries ] + execution_times = classification_df["execution_seconds"].tolist() + assert len(execution_times) == 4 + assert all(isinstance(runtime, float) for runtime in execution_times) captured = capfd.readouterr() assert "Exception in worker" in captured.out diff --git a/packages/phoenix-evals/tests/phoenix/evals/functions/test_executor.py b/packages/phoenix-evals/tests/phoenix/evals/functions/test_executor.py index 7c2ef68d8f..862f1cc36a 100644 --- a/packages/phoenix-evals/tests/phoenix/evals/functions/test_executor.py +++ b/packages/phoenix-evals/tests/phoenix/evals/functions/test_executor.py @@ -107,6 +107,7 @@ async def dummy_fn(payload: int) -> int: outputs, statuses = await executor.execute(inputs) exceptions = [status.exceptions for status in statuses] status_types = [status.status for status in statuses] + execution_times = [status.execution_seconds for status in statuses] assert outputs == [0, 1, 52, 3, 4], "failed tasks use the fallback value" assert [len(excs) if excs else 0 for excs in exceptions] == [ 0, @@ -122,6 +123,8 @@ async def dummy_fn(payload: int) -> int: ExecutionStatus.COMPLETED, ExecutionStatus.COMPLETED, ] + assert len(execution_times) == 5 + assert all(isinstance(runtime, float) for runtime in execution_times) assert all(isinstance(exc, ValueError) for exc in exceptions[2]) @@ -279,6 +282,7 @@ def dummy_fn(payload: int) -> int: outputs, execution_details = executor.run(inputs) exceptions = [status.exceptions for status in execution_details] status_types = [status.status for status in execution_details] + execution_times = [status.execution_seconds for status in execution_details] assert outputs == [0, 1, 52, 3, 4] assert [len(excs) if excs else 0 for excs in exceptions] == [ 0, @@ -294,6 +298,8 @@ def dummy_fn(payload: int) -> int: ExecutionStatus.COMPLETED, ExecutionStatus.COMPLETED, ] + assert len(execution_times) == 5 + assert all(isinstance(runtime, float) for runtime in execution_times) assert all(isinstance(exc, ValueError) for exc in exceptions[2]) diff --git a/tutorials/evals/evaluations_with_error_handling.ipynb b/tutorials/evals/evaluations_with_error_handling.ipynb index dc7f630114..3c6d255f1c 100644 --- a/tutorials/evals/evaluations_with_error_handling.ipynb +++ b/tutorials/evals/evaluations_with_error_handling.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -64,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -93,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -113,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -134,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -169,7 +169,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -179,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -195,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -207,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -216,7 +216,7 @@ "\"Hello! Yes, I'm here and working. How can I assist you today?\"" ] }, - "execution_count": 11, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -229,336 +229,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Default Behavior\n", + "## Checking execution details\n", "\n", - "The default behavior is to retry (with a default maximum of 10) on exceptions while running evals. However, if input data is missing and a prompt cannot be generated from a template, that row will immediately fail. `llm_classify` will return early, and the rows that will not be run will not have an eval." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9d0c104047eb4dde97a4ee43af3c04ca", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "llm_classify | | 0/40 (0.0%) | ⏳ 00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
label
0relevant
1relevant
2unrelated
3unrelated
4unrelated
5unrelated
6relevant
7relevant
8relevant
9unrelated
10relevant
11relevant
12unrelated
13relevant
14relevant
15unrelated
16unrelated
17relevant
18relevant
19unrelated
20relevant
21unrelated
22unrelated
23relevant
24unrelated
25relevant
26None
27None
28None
29None
30None
31None
32None
33None
34None
35None
36None
37None
38None
39None
\n", - "" - ], - "text/plain": [ - " label\n", - "0 relevant\n", - "1 relevant\n", - "2 unrelated\n", - "3 unrelated\n", - "4 unrelated\n", - "5 unrelated\n", - "6 relevant\n", - "7 relevant\n", - "8 relevant\n", - "9 unrelated\n", - "10 relevant\n", - "11 relevant\n", - "12 unrelated\n", - "13 relevant\n", - "14 relevant\n", - "15 unrelated\n", - "16 unrelated\n", - "17 relevant\n", - "18 relevant\n", - "19 unrelated\n", - "20 relevant\n", - "21 unrelated\n", - "22 unrelated\n", - "23 relevant\n", - "24 unrelated\n", - "25 relevant\n", - "26 None\n", - "27 None\n", - "28 None\n", - "29 None\n", - "30 None\n", - "31 None\n", - "32 None\n", - "33 None\n", - "34 None\n", - "35 None\n", - "36 None\n", - "37 None\n", - "38 None\n", - "39 None" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "default_evals" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Including exception details\n", + "The default behavior is to retry (with a default maximum of 10) on exceptions while running evals. However, if input data is missing and a prompt cannot be generated from a template, that row will immediately fail. `llm_classify` will return early, and the rows that will not be run will not have an eval.\n", "\n", - "By setting the `include_exceptions` flag to `True` in `llm_classify`, two additional columns will be provided in the output that will show all exceptions that were encountered during execution, as well as a status that summarizes what happened for each row." + "In addition to the output, columns will be provided in the output that will show all exceptions that were encountered during execution, as well as a status that summarizes what happened for each row and timing info." ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4b984ccb32e342d69273d4b82736c19a", + "model_id": "955d0a9da9cd4e8db2fb9f235078ebeb", "version_major": 2, "version_minor": 0 }, @@ -573,49 +259,29 @@ "name": "stdout", "output_type": "stream", "text": [ - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 3: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 4: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 5: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 6: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 3: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", "Retries exhausted after 1 attempts: Missing template variables: reference\n" ] @@ -635,7 +301,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -662,6 +328,7 @@ " label\n", " exceptions\n", " execution_status\n", + " execution_seconds\n", " \n", " \n", " \n", @@ -670,240 +337,280 @@ " relevant\n", " []\n", " COMPLETED\n", + " 0.541415\n", " \n", " \n", " 1\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.614199\n", " \n", " \n", " 2\n", " unrelated\n", - " []\n", - " COMPLETED\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.526719\n", " \n", " \n", " 3\n", " unrelated\n", " []\n", " COMPLETED\n", + " 0.680465\n", " \n", " \n", " 4\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", + " [RuntimeError('What could have possibly happened here?!')]\n", " COMPLETED WITH RETRIES\n", + " 0.593109\n", " \n", " \n", " 5\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 1.033485\n", " \n", " \n", " 6\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.511282\n", " \n", " \n", " 7\n", - " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", + " unrelated\n", + " [RuntimeError('What could have possibly happened here?!')]\n", " COMPLETED WITH RETRIES\n", + " 0.599838\n", " \n", " \n", " 8\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.712328\n", " \n", " \n", " 9\n", - " unrelated\n", - " []\n", - " COMPLETED\n", + " relevant\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.643351\n", " \n", " \n", " 10\n", - " relevant\n", + " unrelated\n", " []\n", " COMPLETED\n", + " 0.544921\n", " \n", " \n", " 11\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.547776\n", " \n", " \n", " 12\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", + " [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')]\n", " COMPLETED WITH RETRIES\n", + " 0.492994\n", " \n", " \n", " 13\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", + " [RuntimeError('What could have possibly happened here?!')]\n", " COMPLETED WITH RETRIES\n", + " 0.479243\n", " \n", " \n", " 14\n", - " relevant\n", + " unrelated\n", " []\n", " COMPLETED\n", + " 0.629159\n", " \n", " \n", " 15\n", - " unrelated\n", - " []\n", - " COMPLETED\n", + " relevant\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.506721\n", " \n", " \n", " 16\n", - " unrelated\n", + " relevant\n", " []\n", " COMPLETED\n", + " 0.525353\n", " \n", " \n", " 17\n", - " relevant\n", + " unrelated\n", " []\n", " COMPLETED\n", + " 0.550200\n", " \n", " \n", " 18\n", " relevant\n", - " []\n", - " COMPLETED\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.626456\n", " \n", " \n", " 19\n", - " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " relevant\n", + " []\n", + " COMPLETED\n", + " 0.495438\n", " \n", " \n", " 20\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.752875\n", " \n", " \n", " 21\n", - " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " relevant\n", + " []\n", + " COMPLETED\n", + " 0.485587\n", " \n", " \n", " 22\n", " unrelated\n", " []\n", " COMPLETED\n", + " 0.812941\n", " \n", " \n", " 23\n", - " relevant\n", + " None\n", " []\n", - " COMPLETED\n", + " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 24\n", - " unrelated\n", - " []\n", - " COMPLETED\n", + " relevant\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.575621\n", " \n", " \n", " 25\n", - " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " unrelated\n", + " []\n", + " COMPLETED\n", + " 0.817680\n", " \n", " \n", " 26\n", - " None\n", + " unrelated\n", " []\n", - " DID NOT RUN\n", + " COMPLETED\n", + " 0.485074\n", " \n", " \n", " 27\n", " None\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", + " [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')]\n", " DID NOT RUN\n", + " 0.000610\n", " \n", " \n", " 28\n", " None\n", " [PhoenixTemplateMappingError('Missing template variables: reference')]\n", " MISSING INPUT\n", + " 0.000100\n", " \n", " \n", " 29\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 30\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 31\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 32\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 33\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 34\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 35\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 36\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 37\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 38\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", " 39\n", " None\n", " []\n", " DID NOT RUN\n", + " 0.000000\n", " \n", " \n", "\n", @@ -918,26 +625,26 @@ "4 unrelated \n", "5 unrelated \n", "6 relevant \n", - "7 relevant \n", + "7 unrelated \n", "8 relevant \n", - "9 unrelated \n", - "10 relevant \n", + "9 relevant \n", + "10 unrelated \n", "11 relevant \n", "12 unrelated \n", "13 relevant \n", - "14 relevant \n", - "15 unrelated \n", - "16 unrelated \n", - "17 relevant \n", + "14 unrelated \n", + "15 relevant \n", + "16 relevant \n", + "17 unrelated \n", "18 relevant \n", - "19 unrelated \n", + "19 relevant \n", "20 relevant \n", - "21 unrelated \n", + "21 relevant \n", "22 unrelated \n", - "23 relevant \n", - "24 unrelated \n", - "25 relevant \n", - "26 None \n", + "23 None \n", + "24 relevant \n", + "25 unrelated \n", + "26 unrelated \n", "27 None \n", "28 None \n", "29 None \n", @@ -952,92 +659,92 @@ "38 None \n", "39 None \n", "\n", - " exceptions \\\n", - "0 [] \n", - "1 [] \n", - "2 [] \n", - "3 [] \n", - "4 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "5 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "6 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "7 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "8 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "9 [] \n", - "10 [] \n", - "11 [] \n", - "12 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "13 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "14 [] \n", - "15 [] \n", - "16 [] \n", - "17 [] \n", - "18 [] \n", - "19 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "20 [] \n", - "21 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "22 [] \n", - "23 [] \n", - "24 [] \n", - "25 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "26 [] \n", - "27 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "28 [PhoenixTemplateMappingError('Missing template variables: reference')] \n", - "29 [] \n", - "30 [] \n", - "31 [] \n", - "32 [] \n", - "33 [] \n", - "34 [] \n", - "35 [] \n", - "36 [] \n", - "37 [] \n", - "38 [] \n", - "39 [] \n", + " exceptions \\\n", + "0 [] \n", + "1 [] \n", + "2 [RuntimeError('What could have possibly happened here?!')] \n", + "3 [] \n", + "4 [RuntimeError('What could have possibly happened here?!')] \n", + "5 [] \n", + "6 [] \n", + "7 [RuntimeError('What could have possibly happened here?!')] \n", + "8 [] \n", + "9 [RuntimeError('What could have possibly happened here?!')] \n", + "10 [] \n", + "11 [] \n", + "12 [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')] \n", + "13 [RuntimeError('What could have possibly happened here?!')] \n", + "14 [] \n", + "15 [RuntimeError('What could have possibly happened here?!')] \n", + "16 [] \n", + "17 [] \n", + "18 [RuntimeError('What could have possibly happened here?!')] \n", + "19 [] \n", + "20 [] \n", + "21 [] \n", + "22 [] \n", + "23 [] \n", + "24 [RuntimeError('What could have possibly happened here?!')] \n", + "25 [] \n", + "26 [] \n", + "27 [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')] \n", + "28 [PhoenixTemplateMappingError('Missing template variables: reference')] \n", + "29 [] \n", + "30 [] \n", + "31 [] \n", + "32 [] \n", + "33 [] \n", + "34 [] \n", + "35 [] \n", + "36 [] \n", + "37 [] \n", + "38 [] \n", + "39 [] \n", "\n", - " execution_status \n", - "0 COMPLETED \n", - "1 COMPLETED \n", - "2 COMPLETED \n", - "3 COMPLETED \n", - "4 COMPLETED WITH RETRIES \n", - "5 COMPLETED WITH RETRIES \n", - "6 COMPLETED WITH RETRIES \n", - "7 COMPLETED WITH RETRIES \n", - "8 COMPLETED WITH RETRIES \n", - "9 COMPLETED \n", - "10 COMPLETED \n", - "11 COMPLETED \n", - "12 COMPLETED WITH RETRIES \n", - "13 COMPLETED WITH RETRIES \n", - "14 COMPLETED \n", - "15 COMPLETED \n", - "16 COMPLETED \n", - "17 COMPLETED \n", - "18 COMPLETED \n", - "19 COMPLETED WITH RETRIES \n", - "20 COMPLETED \n", - "21 COMPLETED WITH RETRIES \n", - "22 COMPLETED \n", - "23 COMPLETED \n", - "24 COMPLETED \n", - "25 COMPLETED WITH RETRIES \n", - "26 DID NOT RUN \n", - "27 DID NOT RUN \n", - "28 MISSING INPUT \n", - "29 DID NOT RUN \n", - "30 DID NOT RUN \n", - "31 DID NOT RUN \n", - "32 DID NOT RUN \n", - "33 DID NOT RUN \n", - "34 DID NOT RUN \n", - "35 DID NOT RUN \n", - "36 DID NOT RUN \n", - "37 DID NOT RUN \n", - "38 DID NOT RUN \n", - "39 DID NOT RUN " + " execution_status execution_seconds \n", + "0 COMPLETED 0.541415 \n", + "1 COMPLETED 0.614199 \n", + "2 COMPLETED WITH RETRIES 0.526719 \n", + "3 COMPLETED 0.680465 \n", + "4 COMPLETED WITH RETRIES 0.593109 \n", + "5 COMPLETED 1.033485 \n", + "6 COMPLETED 0.511282 \n", + "7 COMPLETED WITH RETRIES 0.599838 \n", + "8 COMPLETED 0.712328 \n", + "9 COMPLETED WITH RETRIES 0.643351 \n", + "10 COMPLETED 0.544921 \n", + "11 COMPLETED 0.547776 \n", + "12 COMPLETED WITH RETRIES 0.492994 \n", + "13 COMPLETED WITH RETRIES 0.479243 \n", + "14 COMPLETED 0.629159 \n", + "15 COMPLETED WITH RETRIES 0.506721 \n", + "16 COMPLETED 0.525353 \n", + "17 COMPLETED 0.550200 \n", + "18 COMPLETED WITH RETRIES 0.626456 \n", + "19 COMPLETED 0.495438 \n", + "20 COMPLETED 0.752875 \n", + "21 COMPLETED 0.485587 \n", + "22 COMPLETED 0.812941 \n", + "23 DID NOT RUN 0.000000 \n", + "24 COMPLETED WITH RETRIES 0.575621 \n", + "25 COMPLETED 0.817680 \n", + "26 COMPLETED 0.485074 \n", + "27 DID NOT RUN 0.000610 \n", + "28 MISSING INPUT 0.000100 \n", + "29 DID NOT RUN 0.000000 \n", + "30 DID NOT RUN 0.000000 \n", + "31 DID NOT RUN 0.000000 \n", + "32 DID NOT RUN 0.000000 \n", + "33 DID NOT RUN 0.000000 \n", + "34 DID NOT RUN 0.000000 \n", + "35 DID NOT RUN 0.000000 \n", + "36 DID NOT RUN 0.000000 \n", + "37 DID NOT RUN 0.000000 \n", + "38 DID NOT RUN 0.000000 \n", + "39 DID NOT RUN 0.000000 " ] }, - "execution_count": 15, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -1055,19 +762,19 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Counter({'COMPLETED': 16,\n", + "Counter({'COMPLETED': 17,\n", " 'DID NOT RUN': 13,\n", - " 'COMPLETED WITH RETRIES': 10,\n", + " 'COMPLETED WITH RETRIES': 9,\n", " 'MISSING INPUT': 1})" ] }, - "execution_count": 16, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -1087,13 +794,13 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e3a643667e174b48800a321a87223a81", + "model_id": "571cca2fc79d4655b554a245dddb5676", "version_major": 2, "version_minor": 0 }, @@ -1108,51 +815,33 @@ "name": "stdout", "output_type": "stream", "text": [ - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Retries exhausted after 3 attempts: What could have possibly happened here?!\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", "Retries exhausted after 1 attempts: Missing template variables: reference\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", + "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!')\n", "Requeuing...\n", - "Exception in worker on attempt 2: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Retries exhausted after 3 attempts: What could have possibly happened here?!?!?!\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n", - "Retries exhausted after 1 attempts: Missing template variables: input\n", - "Exception in worker on attempt 1: raised RuntimeError('What could have possibly happened here?!?!?!')\n", - "Requeuing...\n" + "Retries exhausted after 1 attempts: Missing template variables: input\n" ] } ], @@ -1165,14 +854,13 @@ " rails=rails,\n", " concurrency=3,\n", " max_retries=2,\n", - " include_exceptions=True,\n", " exit_on_error=False,\n", ")" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -1199,248 +887,289 @@ " label\n", " exceptions\n", " execution_status\n", + " execution_seconds\n", " \n", " \n", " \n", " \n", " 0\n", " relevant\n", - " []\n", - " COMPLETED\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.656291\n", " \n", " \n", " 1\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 1.214457\n", " \n", " \n", " 2\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 1.006221\n", " \n", " \n", " 3\n", - " unrelated\n", - " []\n", - " COMPLETED\n", + " None\n", + " [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')]\n", + " FAILED\n", + " 0.000806\n", " \n", " \n", " 4\n", " unrelated\n", " []\n", " COMPLETED\n", + " 0.598868\n", " \n", " \n", " 5\n", " unrelated\n", - " []\n", - " COMPLETED\n", + " [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.620612\n", " \n", " \n", " 6\n", " relevant\n", - " []\n", - " COMPLETED\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.491993\n", " \n", " \n", " 7\n", - " relevant\n", + " unrelated\n", " []\n", " COMPLETED\n", + " 0.498494\n", " \n", " \n", " 8\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.460664\n", " \n", " \n", " 9\n", - " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " relevant\n", + " []\n", + " COMPLETED\n", + " 0.543839\n", " \n", " \n", " 10\n", - " relevant\n", + " unrelated\n", " []\n", " COMPLETED\n", + " 0.361752\n", " \n", " \n", " 11\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.827295\n", " \n", " \n", " 12\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.565072\n", " \n", " \n", " 13\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.441669\n", " \n", " \n", " 14\n", - " relevant\n", + " unrelated\n", " []\n", " COMPLETED\n", + " 0.860275\n", " \n", " \n", " 15\n", - " unrelated\n", + " relevant\n", " []\n", " COMPLETED\n", + " 0.361320\n", " \n", " \n", " 16\n", - " unrelated\n", + " relevant\n", " []\n", " COMPLETED\n", + " 0.684321\n", " \n", " \n", " 17\n", - " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " unrelated\n", + " []\n", + " COMPLETED\n", + " 0.595008\n", " \n", " \n", " 18\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.480569\n", " \n", " \n", " 19\n", - " unrelated\n", - " []\n", - " COMPLETED\n", + " relevant\n", + " [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.658973\n", " \n", " \n", " 20\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.672698\n", " \n", " \n", " 21\n", - " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", + " relevant\n", + " [RuntimeError('What could have possibly happened here?!')]\n", " COMPLETED WITH RETRIES\n", + " 0.372545\n", " \n", " \n", " 22\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.506847\n", " \n", " \n", " 23\n", - " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", - " \n", - " \n", - " 24\n", " unrelated\n", " []\n", " COMPLETED\n", + " 0.670335\n", " \n", " \n", - " 25\n", + " 24\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.545959\n", " \n", " \n", - " 26\n", + " 25\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", + " [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')]\n", " COMPLETED WITH RETRIES\n", + " 0.515250\n", + " \n", + " \n", + " 26\n", + " unrelated\n", + " []\n", + " COMPLETED\n", + " 0.585052\n", " \n", " \n", " 27\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.518192\n", " \n", " \n", " 28\n", " None\n", " [PhoenixTemplateMappingError('Missing template variables: reference')]\n", " MISSING INPUT\n", + " 0.000257\n", " \n", " \n", " 29\n", " relevant\n", " []\n", " COMPLETED\n", + " 0.478919\n", " \n", " \n", " 30\n", - " unrelated\n", + " relevant\n", " []\n", " COMPLETED\n", + " 0.522542\n", " \n", " \n", " 31\n", - " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " relevant\n", + " []\n", + " COMPLETED\n", + " 0.643352\n", " \n", " \n", " 32\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.471015\n", " \n", " \n", " 33\n", - " None\n", - " [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')]\n", - " FAILED\n", + " relevant\n", + " [RuntimeError('What could have possibly happened here?!')]\n", + " COMPLETED WITH RETRIES\n", + " 0.485109\n", " \n", " \n", " 34\n", - " relevant\n", + " unrelated\n", " []\n", " COMPLETED\n", + " 0.384154\n", " \n", " \n", " 35\n", - " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " relevant\n", + " []\n", + " COMPLETED\n", + " 0.481911\n", " \n", " \n", " 36\n", " unrelated\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.486317\n", " \n", " \n", " 37\n", " None\n", " [PhoenixTemplateMappingError('Missing template variables: input')]\n", " MISSING INPUT\n", + " 0.000209\n", " \n", " \n", " 38\n", " unrelated\n", " []\n", " COMPLETED\n", + " 0.580683\n", " \n", " \n", " 39\n", " relevant\n", - " [RuntimeError('What could have possibly happened here?!?!?!')]\n", - " COMPLETED WITH RETRIES\n", + " []\n", + " COMPLETED\n", + " 0.377904\n", " \n", " \n", "\n", @@ -1451,130 +1180,130 @@ "0 relevant \n", "1 relevant \n", "2 unrelated \n", - "3 unrelated \n", + "3 None \n", "4 unrelated \n", "5 unrelated \n", "6 relevant \n", - "7 relevant \n", + "7 unrelated \n", "8 relevant \n", - "9 unrelated \n", - "10 relevant \n", + "9 relevant \n", + "10 unrelated \n", "11 relevant \n", "12 unrelated \n", "13 relevant \n", - "14 relevant \n", - "15 unrelated \n", - "16 unrelated \n", - "17 relevant \n", + "14 unrelated \n", + "15 relevant \n", + "16 relevant \n", + "17 unrelated \n", "18 relevant \n", - "19 unrelated \n", + "19 relevant \n", "20 relevant \n", - "21 unrelated \n", + "21 relevant \n", "22 unrelated \n", - "23 relevant \n", - "24 unrelated \n", - "25 relevant \n", + "23 unrelated \n", + "24 relevant \n", + "25 unrelated \n", "26 unrelated \n", "27 relevant \n", "28 None \n", "29 relevant \n", - "30 unrelated \n", - "31 unrelated \n", + "30 relevant \n", + "31 relevant \n", "32 relevant \n", - "33 None \n", - "34 relevant \n", - "35 unrelated \n", + "33 relevant \n", + "34 unrelated \n", + "35 relevant \n", "36 unrelated \n", "37 None \n", "38 unrelated \n", "39 relevant \n", "\n", - " exceptions \\\n", - "0 [] \n", - "1 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "2 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "3 [] \n", - "4 [] \n", - "5 [] \n", - "6 [] \n", - "7 [] \n", - "8 [] \n", - "9 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "10 [] \n", - "11 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "12 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "13 [] \n", - "14 [] \n", - "15 [] \n", - "16 [] \n", - "17 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "18 [] \n", - "19 [] \n", - "20 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "21 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "22 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "23 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "24 [] \n", - "25 [] \n", - "26 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "27 [] \n", - "28 [PhoenixTemplateMappingError('Missing template variables: reference')] \n", - "29 [] \n", - "30 [] \n", - "31 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "32 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "33 [RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!'), RuntimeError('What could have possibly happened here?!?!?!')] \n", - "34 [] \n", - "35 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "36 [RuntimeError('What could have possibly happened here?!?!?!')] \n", - "37 [PhoenixTemplateMappingError('Missing template variables: input')] \n", - "38 [] \n", - "39 [RuntimeError('What could have possibly happened here?!?!?!')] \n", + " exceptions \\\n", + "0 [RuntimeError('What could have possibly happened here?!')] \n", + "1 [] \n", + "2 [] \n", + "3 [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')] \n", + "4 [] \n", + "5 [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')] \n", + "6 [RuntimeError('What could have possibly happened here?!')] \n", + "7 [] \n", + "8 [] \n", + "9 [] \n", + "10 [] \n", + "11 [] \n", + "12 [] \n", + "13 [] \n", + "14 [] \n", + "15 [] \n", + "16 [] \n", + "17 [] \n", + "18 [] \n", + "19 [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')] \n", + "20 [] \n", + "21 [RuntimeError('What could have possibly happened here?!')] \n", + "22 [] \n", + "23 [] \n", + "24 [] \n", + "25 [RuntimeError('What could have possibly happened here?!'), RuntimeError('What could have possibly happened here?!')] \n", + "26 [] \n", + "27 [] \n", + "28 [PhoenixTemplateMappingError('Missing template variables: reference')] \n", + "29 [] \n", + "30 [] \n", + "31 [] \n", + "32 [] \n", + "33 [RuntimeError('What could have possibly happened here?!')] \n", + "34 [] \n", + "35 [] \n", + "36 [] \n", + "37 [PhoenixTemplateMappingError('Missing template variables: input')] \n", + "38 [] \n", + "39 [] \n", "\n", - " execution_status \n", - "0 COMPLETED \n", - "1 COMPLETED WITH RETRIES \n", - "2 COMPLETED WITH RETRIES \n", - "3 COMPLETED \n", - "4 COMPLETED \n", - "5 COMPLETED \n", - "6 COMPLETED \n", - "7 COMPLETED \n", - "8 COMPLETED \n", - "9 COMPLETED WITH RETRIES \n", - "10 COMPLETED \n", - "11 COMPLETED WITH RETRIES \n", - "12 COMPLETED WITH RETRIES \n", - "13 COMPLETED \n", - "14 COMPLETED \n", - "15 COMPLETED \n", - "16 COMPLETED \n", - "17 COMPLETED WITH RETRIES \n", - "18 COMPLETED \n", - "19 COMPLETED \n", - "20 COMPLETED WITH RETRIES \n", - "21 COMPLETED WITH RETRIES \n", - "22 COMPLETED WITH RETRIES \n", - "23 COMPLETED WITH RETRIES \n", - "24 COMPLETED \n", - "25 COMPLETED \n", - "26 COMPLETED WITH RETRIES \n", - "27 COMPLETED \n", - "28 MISSING INPUT \n", - "29 COMPLETED \n", - "30 COMPLETED \n", - "31 COMPLETED WITH RETRIES \n", - "32 COMPLETED WITH RETRIES \n", - "33 FAILED \n", - "34 COMPLETED \n", - "35 COMPLETED WITH RETRIES \n", - "36 COMPLETED WITH RETRIES \n", - "37 MISSING INPUT \n", - "38 COMPLETED \n", - "39 COMPLETED WITH RETRIES " + " execution_status execution_seconds \n", + "0 COMPLETED WITH RETRIES 0.656291 \n", + "1 COMPLETED 1.214457 \n", + "2 COMPLETED 1.006221 \n", + "3 FAILED 0.000806 \n", + "4 COMPLETED 0.598868 \n", + "5 COMPLETED WITH RETRIES 0.620612 \n", + "6 COMPLETED WITH RETRIES 0.491993 \n", + "7 COMPLETED 0.498494 \n", + "8 COMPLETED 0.460664 \n", + "9 COMPLETED 0.543839 \n", + "10 COMPLETED 0.361752 \n", + "11 COMPLETED 0.827295 \n", + "12 COMPLETED 0.565072 \n", + "13 COMPLETED 0.441669 \n", + "14 COMPLETED 0.860275 \n", + "15 COMPLETED 0.361320 \n", + "16 COMPLETED 0.684321 \n", + "17 COMPLETED 0.595008 \n", + "18 COMPLETED 0.480569 \n", + "19 COMPLETED WITH RETRIES 0.658973 \n", + "20 COMPLETED 0.672698 \n", + "21 COMPLETED WITH RETRIES 0.372545 \n", + "22 COMPLETED 0.506847 \n", + "23 COMPLETED 0.670335 \n", + "24 COMPLETED 0.545959 \n", + "25 COMPLETED WITH RETRIES 0.515250 \n", + "26 COMPLETED 0.585052 \n", + "27 COMPLETED 0.518192 \n", + "28 MISSING INPUT 0.000257 \n", + "29 COMPLETED 0.478919 \n", + "30 COMPLETED 0.522542 \n", + "31 COMPLETED 0.643352 \n", + "32 COMPLETED 0.471015 \n", + "33 COMPLETED WITH RETRIES 0.485109 \n", + "34 COMPLETED 0.384154 \n", + "35 COMPLETED 0.481911 \n", + "36 COMPLETED 0.486317 \n", + "37 MISSING INPUT 0.000209 \n", + "38 COMPLETED 0.580683 \n", + "39 COMPLETED 0.377904 " ] }, - "execution_count": 18, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -1592,19 +1321,19 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Counter({'COMPLETED': 21,\n", - " 'COMPLETED WITH RETRIES': 16,\n", + "Counter({'COMPLETED': 30,\n", + " 'COMPLETED WITH RETRIES': 7,\n", " 'MISSING INPUT': 2,\n", " 'FAILED': 1})" ] }, - "execution_count": 19, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" }