Merge branch 'branch-25.02' of github.com:nv-morpheus/Morpheus into d…

…avid-arm-ci

.devcontainer/Dockerfile

@@ -16,9 +16,9 @@
 ARG BASE
 ARG PYTHON_PACKAGE_MANAGER
 
-FROM node:22 as node
+FROM node:22 AS node
 
-FROM ${BASE} as base
+FROM ${BASE} AS base
 
 # ===== install common packages ==================================================================
 
@@ -71,11 +71,11 @@ npm install -g camouflage-server@0.15
 EOF
 
 
-FROM base as pip-base
+FROM base AS pip-base
 
 ENV DEFAULT_VIRTUAL_ENV=morpheus
 
-FROM base as conda-base
+FROM base AS conda-base
 
 ENV DEFAULT_CONDA_ENV=morpheus
 

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
                 additional_dependencies: [pyproject-flake8]
                 args: ["--config=./pyproject.toml"]
       - repo: https://github.com/google/yapf
-        rev: v0.40.2
+        rev: v0.43.0
         hooks:
               - id: yapf
                 args: ["--style", "./pyproject.toml"]

ci/release/update-version.sh

@@ -82,6 +82,7 @@ sed_runner "s|branch-${CURRENT_SHORT_TAG}|branch-${NEXT_SHORT_TAG}|g" manifest.y
 
 # Depedencies file
 sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" dependencies.yaml
+sed_runner "s/morpheus-dfp=${CURRENT_SHORT_TAG}/morpheus-dfp=${NEXT_SHORT_TAG}/g" dependencies.yaml
 
 # Generate the environment files based upon the updated dependencies.yaml
 rapids-dependency-file-generator

conda/environments/all_cuda-125_arch-aarch64.yaml

@@ -119,7 +119,7 @@ dependencies:
 - versioneer-518
 - watchdog=3.0
 - websockets
-- yapf=0.40.1
+- yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - --find-links https://data.dgl.ai/wheels-test/repo.html

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -125,7 +125,7 @@ dependencies:
 - versioneer-518
 - watchdog=3.0
 - websockets
-- yapf=0.40.1
+- yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - --find-links https://data.dgl.ai/wheels-test/repo.html

conda/environments/dev_cuda-125_arch-aarch64.yaml

@@ -100,7 +100,7 @@ dependencies:
 - versioneer-518
 - watchdog=3.0
 - websockets
-- yapf=0.40.1
+- yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - databricks-cli < 0.100

conda/environments/dev_cuda-125_arch-x86_64.yaml

@@ -105,7 +105,7 @@ dependencies:
 - versioneer-518
 - watchdog=3.0
 - websockets
-- yapf=0.40.1
+- yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - databricks-cli < 0.100

dependencies.yaml

@@ -334,7 +334,7 @@ dependencies:
           - pylint=3.0.3
           - pynvml=11.4
           - versioneer
-          - yapf=0.40.1
+          - yapf=0.43
     specific:
       - output_types: [conda]
         matrices:
@@ -509,7 +509,7 @@ dependencies:
     common:
       - output_types: [conda]
         packages:
-          - morpheus-dfp=24.10
+          - morpheus-dfp=25.02
           - tini=0.19
           - pip:
             - *torch-extra-index

docker/Dockerfile

@@ -123,7 +123,7 @@ RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
 
 # ============ Stage: conda_env ============
 # Create the conda environment and install all dependencies
-FROM base as conda_env
+FROM base AS conda_env
 
 ARG MORPHEUS_ROOT_HOST
 
@@ -151,7 +151,7 @@ SHELL ["/bin/bash", "-c"]
 
 # ============ Stage: conda_bld_morpheus_core ============
 # Now build the morpheus conda package
-FROM conda_env as conda_bld_morpheus_core
+FROM conda_env AS conda_bld_morpheus_core
 
 ARG MORPHEUS_ROOT_HOST
 ARG CUDA_MAJOR_VER
@@ -198,7 +198,7 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke
 
 # ============ Stage: base_extended ============
 # Base environment with any optional extensions. ex. DOCA libs
-FROM conda_env as base_extended
+FROM conda_env AS base_extended
 
 # Add one or more optional dependencies to the base environment
 ARG MORPHEUS_ROOT_HOST
@@ -216,7 +216,7 @@ RUN --mount=type=cache,id=doca,target=/tmp/doca,sharing=locked \
 
 # ============ Stage: conda_env_dev ============
 # Create the development conda environment
-FROM base_extended as conda_env_dev
+FROM base_extended AS conda_env_dev
 
 ARG MORPHEUS_ROOT_HOST
 ARG CUDA_MAJOR_VER
@@ -237,7 +237,7 @@ RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
 
 # ============ Stage: conda_bld_morpheus ============
 # Now build the morpheus conda package
-FROM base_extended as conda_bld_morpheus
+FROM base_extended AS conda_bld_morpheus
 
 ARG MORPHEUS_ROOT_HOST
 ARG CUDA_MAJOR_VER
@@ -278,7 +278,7 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke
 
 # ============ Stage: build_docs ============
 # Now build the morpheus docs
-FROM conda_env_dev as build_docs
+FROM conda_env_dev AS build_docs
 
 ARG MORPHEUS_ROOT_HOST
 
@@ -308,7 +308,7 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke
 
 # ============ Stage: runtime_conda_create ============
 # Setup conda for the runtime environment
-FROM base_extended as runtime_conda_create
+FROM base_extended AS runtime_conda_create
 ARG MORPHEUS_ROOT_HOST
 ARG CUDA_MAJOR_VER
 ARG CUDA_MINOR_VER
@@ -341,7 +341,7 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target
 # ============ Stage: git_clone ============
 # Perform a clone of the git repository this ensures that when we copy files from the source repository, we aren't
 # unintentionally including build artifacts or other files that shouldn't be in the final image
-FROM conda_env_dev as git_clone
+FROM conda_env_dev AS git_clone
 
 ARG MORPHEUS_ROOT_HOST
 
@@ -356,7 +356,7 @@ RUN --mount=type=bind,source=.,target=/opt/host_repo \
 
 # ============ Stage: runtime ============
 # Setup container for runtime environment
-FROM runtime_conda_create as runtime
+FROM runtime_conda_create AS runtime
 
 ARG MORPHEUS_ROOT_HOST
 
@@ -380,7 +380,7 @@ CMD [ "morpheus" ]
 
 # ============ Stage: development ============
 # Install and configure development only packages
-FROM conda_env_dev as development
+FROM conda_env_dev AS development
 
 ARG CUDA_MAJOR_VER
 ARG CUDA_MINOR_VER
@@ -412,7 +412,7 @@ RUN git config --global --add safe.directory "*"
 
 # ============ Stage: python_debug_bld ===========
 # Configure and build cpython with debugging symbols
-FROM development as development_pydbg
+FROM development AS development_pydbg
 
 COPY ${MORPHEUS_ROOT_HOST}/ci/conda/recipes/python-dbg/ ./ci/conda/recipes/python-dbg
 COPY ${MORPHEUS_ROOT_HOST}/ci/conda/recipes/run_conda_build.sh ./ci/conda/recipes/run_conda_build.sh

docs/source/developer_guide/guides/5_digital_fingerprinting.md

@@ -129,6 +129,7 @@ The reference architecture is composed of the following services:​
 | `mlflow` | [MLflow](https://mlflow.org/) provides a versioned model store​ |
 | `jupyter` | [Jupyter Server](https://jupyter-server.readthedocs.io/en/latest/)​ necessary for testing and development of the pipelines​ |
 | `morpheus_pipeline` | Used for executing both training and inference pipelines |
+| `fetch_data` | Downloads the example datasets for the DFP example |
 
 ### Running via `docker-compose`
 #### System requirements
@@ -158,12 +159,32 @@ docker compose build
 #### Downloading the example datasets
 First, we will need to install additional requirements in to the Conda environment. Then run the `examples/digital_fingerprinting/fetch_example_data.py` script. This will download the example data into the `examples/data/dfp` dir.
 
-From the Morpheus repo, run:
+The script can be run from within the `fetch_data` Docker Compose service, or from within a Conda environment on the host machine.
+
+##### Docker Compose Service Method
+This approach has the advantage of not requiring any additional setup on the host machine. From the `examples/digital_fingerprinting/production` dir run:
+```bash
+docker compose up mlflow
+```
+##### Conda Environment Method
+This approach is useful for users who have already set up a Conda environment on their host machine, and has the advantage that the downloaded data will be owned by the host user.
+
+If a Conda environment has already been created, it can be updated by running the following command from the root of the Morpheus repo:
 ```bash
 conda env update --solver=libmamba \
   -n ${CONDA_DEFAULT_ENV} \
   --file ./conda/environments/examples_cuda-125_arch-x86_64.yaml
+```
 
+If a Conda environment has not been created, it can be created by running the following command from the root of the Morpheus repo:
+```bash
+conda env create --solver=libmamba \
+  -n morpheus \
+  --file ./conda/environments/all_cuda-125_arch-x86_64.yaml
+```
+
+Once the Conda environment has been updated or created, fetch the data with the following command:
+```bash
 python examples/digital_fingerprinting/fetch_example_data.py all
 ```
 

examples/digital_fingerprinting/fetch_example_data.py

@@ -543,6 +543,8 @@ def fetch_dataset(dataset):
     fs_hndl = s3fs.S3FileSystem(anon=True)
     s3_base_path = os.path.join(S3_BASE_PATH, dataset)
 
+    download_count = 0
+
     train_dir = f"{EXAMPLE_DATA_DIR}/dfp/{dataset}-training-data/"
     if not os.path.exists(train_dir):
         os.makedirs(train_dir)
@@ -552,6 +554,7 @@ def fetch_dataset(dataset):
         if not exists(train_dir + f):
             print(f"Downloading {f}")
             fs_hndl.get_file(os.path.join(s3_base_path, f), train_dir + f)
+            download_count += 1
 
     infer_dir = f"{EXAMPLE_DATA_DIR}/dfp/{dataset}-inference-data/"
     if not exists(infer_dir):
@@ -562,6 +565,12 @@ def fetch_dataset(dataset):
         if not os.path.exists(infer_dir + f):
             print(f"Downloading {f}")
             fs_hndl.get_file(os.path.join(s3_base_path, f), infer_dir + f)
+            download_count += 1
+
+    if download_count == 0:
+        print(f"No new files to download for {dataset} dataset")
+    else:
+        print(f"Download complete for {dataset} dataset")
 
 
 def parse_args():

examples/digital_fingerprinting/production/Dockerfile

@@ -16,7 +16,7 @@
 ARG BASE_IMG=nvcr.io/nvidia/cuda
 ARG BASE_IMG_TAG=12.5.1-base-ubuntu22.04
 
-FROM ${BASE_IMG}:${BASE_IMG_TAG} as base
+FROM ${BASE_IMG}:${BASE_IMG_TAG} AS base
 
 # Install necessary dependencies using apt-get
 RUN apt-get update && apt-get install -y \
@@ -55,13 +55,13 @@ ENTRYPOINT [ "/opt/conda/envs/morpheus-dfp/bin/tini", "--", "/workspace/examples
 SHELL ["/bin/bash", "-c"]
 
 # ===== Setup for running unattended =====
-FROM base as runtime
+FROM base AS runtime
 
 # Launch morpheus
 CMD ["./launch.sh"]
 
 # ===== Setup for running Jupyter =====
-FROM base as jupyter
+FROM base AS jupyter
 
 # Install the jupyter specific requirements
 RUN source activate morpheus-dfp &&\

examples/digital_fingerprinting/production/README.md

@@ -44,7 +44,32 @@ docker compose build
 >
 > This is most likely due to using an older version of the `docker-compose` command, instead re-run the build with `docker compose`. Refer to [Migrate to Compose V2](https://docs.docker.com/compose/migrate/) for more information.
 
+### Fetch Example Data
+The `examples/digital_fingerprinting/fetch_example_data.py` script can be used to fetch the Duo and Azure logs to run the example pipelines.
+
+Download the data needed to run the DFP pipeline on Azure / Duo logs:
+```bash
+docker compose run fetch_data
+```
+
+
 ### Running the services
+
+The Morpheus DFP pipeline can be run from either a Jupyter Notebook using the `jupyter` service or from the command line using the `morpheus_pipeline` service. The `mlflow` service is also started in the background to provide a tracking URI for the Morpheus pipeline.
+
+#### Optional MLflow Service
+Starting either the `morpheus_pipeline` or the `jupyter` service, will start the `mlflow` service in the background. For debugging purposes it can be helpful to view the logs of the running MLflow service.
+
+From the `examples/digital_fingerprinting/production` dir run:
+```bash
+docker compose up mlflow
+```
+
+By default, a MLflow dashboard will be available at:
+```bash
+http://localhost:5000
+```
+
 #### Jupyter Server
 From the `examples/digital_fingerprinting/production` dir run:
 ```bash
@@ -70,7 +95,7 @@ Copy and paste the URL into a web browser. There are six notebooks included with
 
 > **Note:** The token in the URL is a one-time use token, and a new one is generated with each invocation.
 
-#### Morpheus Pipeline
+#### Morpheus Pipeline Service
 By default the `morpheus_pipeline` will run the training pipeline for Duo data, from the `examples/digital_fingerprinting/production` dir run:
 ```bash
 docker compose up morpheus_pipeline
@@ -105,29 +130,6 @@ Both scripts are capable of running either a training or inference pipeline for
 | `--help` | | Show this message and exit. |
 
 ##### Steps to Run Example Pipeline
-The `examples/digital_fingerprinting/fetch_example_data.py` script can be used to fetch the Duo and Azure logs to run the example pipelines.
-
-```bash
-export DFP_HOME=examples/digital_fingerprinting
-```
-
-Usage of the script is as follows:
-```bash
-python $DFP_HOME/fetch_example_data.py --help
-
-usage: Fetches training and inference data for DFP examples [-h] [{azure,duo,all} [{azure,duo,all} ...]]
-
-positional arguments:
-  {azure,duo,all}  Data set to fetch
-
-optional arguments:
-  -h, --help       show this help message and exit
-```
-
-Download the data needed to run a pipeline on Azure / Duo logs:
-```bash
-python $DFP_HOME/fetch_example_data.py all
-```
 
 Run Duo Training Pipeline:
 ```bash
@@ -156,19 +158,6 @@ The commands in the previous section run stage-based example DFP pipelines. The
 
 Commands to run equivalent module-based DFP pipelines can be found [here](../../../docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md#running-example-modular-dfp-pipelines).
 
-#### Optional MLflow Service
-Starting either the `morpheus_pipeline` or the `jupyter` service, will start the `mlflow` service in the background. For debugging purposes it can be helpful to view the logs of the running MLflow service.
-
-From the `examples/digital_fingerprinting/production` dir run:
-```bash
-docker compose up mlflow
-```
-
-By default, a MLflow dashboard will be available at:
-```bash
-http://localhost:5000
-```
-
 ## Kubernetes deployment
 
 The Morpheus project also maintains Helm charts and container images for Kubernetes deployment of Morpheus and MLflow (both for serving and for the Triton plugin). These are located in the NVIDIA GPU Cloud (NGC) [public catalog](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/morpheus/collections/morpheus_).

examples/digital_fingerprinting/production/conda/environments/dfp_example_cuda-125_arch-aarch64.yaml

@@ -11,7 +11,7 @@ channels:
 dependencies:
 - boto3=1.35
 - kfp
-- morpheus-dfp=24.10
+- morpheus-dfp=25.02
 - nodejs=18.*
 - papermill=2.4.0
 - pip