Dev Environments with Docker Compose

.gitignore

@@ -20,3 +20,6 @@ __pycache__/
 /.*_checkpoints/
 .ipynb_checkpoints/*
 */.jupyter/*
+
+.bash_history
+.ipython/

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "merlin/systems"]
+	path = merlin/systems
+	url = git@github.com:NVIDIA-Merlin/systems.git
+[submodule "merlin/NVTabular"]
+	path = merlin/NVTabular
+	url = git@github.com:NVIDIA-Merlin/NVTabular.git
+[submodule "merlin/core"]
+	path = merlin/core
+	url = git@github.com:NVIDIA-Merlin/core.git
+[submodule "merlin/models"]
+	path = merlin/models
+	url = git@github.com:NVIDIA-Merlin/models.git
+[submodule "merlin/dataloader"]
+	path = merlin/dataloader
+	url = git@github.com:NVIDIA-Merlin/dataloader.git
+[submodule "merlin/Transformers4Rec"]
+	path = merlin/Transformers4Rec
+	url = git@github.com:NVIDIA-Merlin/Transformers4Rec.git

docker-compose.yml

@@ -0,0 +1,53 @@
+---
+services:
+
+  dev-cpu:
+    build:
+      context: .
+      dockerfile: docker/dev.cpu.Dockerfile
+    volumes:
+      - .:/workspace
+    working_dir: "/workspace"
+    ports:
+      - "8888:8888"
+    command: "jupyter notebook --ip 0.0.0.0 --allow-root"
+    environment:
+      HISTFILE: "/workspace/.bash_history"
+      IPYTHONDIR: "/workspace/.ipython"
+      BASE_DIR: "/workspace/base"
+      FEAST_USAGE: "False"  # disale usage logging
+      TF_CPP_MIN_LOG_LEVEL: "1"  # disable INFO logging
+    shm_size: '1gb'
+
+  dev-gpu:
+    build:
+      context: .
+      dockerfile: docker/dev.gpu.Dockerfile
+    volumes:
+      - .:/workspace
+    working_dir: "/workspace"
+    ports:
+      - "8888:8888"
+    command: "jupyter notebook --ip 0.0.0.0 --allow-root"
+    entrypoint: "/opt/nvidia/nvidia_entrypoint.sh"
+    environment:
+      HISTFILE: "/workspace/.bash_history"
+      IPYTHONDIR: "/workspace/.ipython"
+      BASE_DIR: "/workspace/base"
+      FEAST_USAGE: "False"  # disale usage logging
+      TF_CPP_MIN_LOG_LEVEL: "1"  # disable INFO logging
+    shm_size: '1gb'
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - capabilities: [gpu]
+
+  requirements:
+    build:
+      context: .
+      dockerfile: docker/requirements.Dockerfile
+    volumes:
+      - ./requirements:/workspace
+      - ./merlin:/workspace/merlin
+    working_dir: "/workspace"

docker/dev.cpu.Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.8.16-slim-buster
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    build-essential `For building NVTabular C++ code` \
+    graphviz `# For visualizing merlin graphs` \
+    ripgrep
+
+# Update pip
+RUN pip install --no-cache-dir --upgrade \
+    pip \
+    virtualenv
+
+# -----------------------------------------------------------------------------
+# Setup Virtualenv
+# -----------------------------------------------------------------------------
+
+# The virtual env is required to allow us to install the merlin packages
+# in development (editable) mode succesfully
+
+ENV VIRTUALENV_PATH=/venv
+RUN python -m virtualenv $VIRTUALENV_PATH
+ENV PATH="${VIRTUALENV_PATH}/bin:${PATH}"
+
+WORKDIR /workspace
+
+# install common dev requirements
+COPY requirements/dev.cpu.txt /tmp/requirements.txt
+RUN python -m pip install --no-cache-dir -r /tmp/requirements.txt
+
+# -----------------------------------------------------------------------------
+# Install Merlin Repos
+# -----------------------------------------------------------------------------
+
+COPY merlin merlin
+
+# install merlin packages in development mode
+RUN python -m pip install -e merlin/models --no-deps
+RUN python -m pip install -e merlin/core --no-deps
+RUN python -m pip install -e merlin/NVTabular --no-deps
+RUN python -m pip install -e merlin/systems --no-deps
+RUN python -m pip install -e merlin/dataloader --no-deps
+RUN python -m pip install -e merlin/Transformers4Rec --no-deps

docker/dev.gpu.Dockerfile

@@ -0,0 +1,63 @@
+FROM nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04
+
+# -----------------------------------------------------------------------------
+# Install system packages
+# -----------------------------------------------------------------------------
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    software-properties-common \
+    graphviz `# For visualizing merlin graphs` \
+    libcudnn8 \
+    ripgrep
+
+RUN add-apt-repository ppa:deadsnakes/ppa -y
+
+RUN apt-get update && TZ=Etc/UTC apt install -y \
+    python3.8 \
+    python3.8-dev \
+    python3.8-distutils
+
+RUN ln -s /usr/bin/python3.8 /usr/bin/python
+
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py
+
+# Update pip
+RUN pip install --no-cache-dir --upgrade \
+    pip \
+    virtualenv
+
+# -----------------------------------------------------------------------------
+# Setup Virtualenv
+# -----------------------------------------------------------------------------
+
+# The virtual env is required to allow us to install the merlin packages
+# in development (editable) mode succesfully
+
+ENV VIRTUALENV_PATH=/venv
+RUN python -m virtualenv $VIRTUALENV_PATH
+ENV PATH="${VIRTUALENV_PATH}/bin:${PATH}"
+
+WORKDIR /workspace
+
+# install common dev requirements
+COPY requirements/dev.gpu.txt /tmp/requirements.txt
+RUN python -m pip install --no-cache-dir -r /tmp/requirements.txt
+
+# -----------------------------------------------------------------------------
+# Install Merlin Repos
+# -----------------------------------------------------------------------------
+
+COPY merlin merlin
+
+# install merlin packages in development mode
+RUN python -m pip install -e merlin/models --no-deps
+RUN python -m pip install -e merlin/core --no-deps
+RUN python -m pip install -e merlin/NVTabular --no-deps
+RUN python -m pip install -e merlin/systems --no-deps
+RUN python -m pip install -e merlin/dataloader --no-deps
+RUN python -m pip install -e merlin/Transformers4Rec --no-deps

docker/dev.tf.Dockerfile

@@ -0,0 +1,74 @@
+FROM nvcr.io/nvidia/merlin/merlin-tensorflow:22.12
+
+# -----------------------------------------------------------------------------
+# Install system packages
+# -----------------------------------------------------------------------------
+
+RUN apt-get update
+RUN apt-get install -y \
+    graphviz `# For visualizing merlin graphs` \
+    sudo `# for installing deps from git repos`
+
+# Update pip
+RUN pip install --upgrade \
+    pip \
+    virtualenv
+
+# remove default installation of merlin packages
+RUN pip uninstall -y \
+    merlin-core \
+    merlin-models \
+    merlin-systems \
+    nvtabular \
+    transformers4rec \
+    ipython \
+    jupyter
+
+# -----------------------------------------------------------------------------
+# Setup User
+# -----------------------------------------------------------------------------
+
+ARG UID
+ARG GID
+ARG USER
+
+RUN addgroup --gid $GID $USER
+RUN useradd --no-log-init --no-user-group --create-home --home-dir /home/$USER --password '' --uid $UID --gid $GID $USER
+RUN usermod -a -G sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER $USER
+
+# -----------------------------------------------------------------------------
+# Setup Virtualenv
+# -----------------------------------------------------------------------------
+
+# The virtual env is required to allow us to install the merlin packages
+# in development (editable) mode succesfully
+
+ENV VIRTUALENV_PATH=/home/$USER/venv
+RUN python -m virtualenv $VIRTUALENV_PATH
+ENV PATH="${VIRTUALENV_PATH}/bin:${PATH}"
+
+# Extending the PYTHONPATH so that we can continue to find packages already installed
+# e.g 'cudf', 'cuda-python' and others in /usr/local/lib/
+ENV PYTHONPATH="${VIRTUALENV_PATH}/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages"
+
+WORKDIR /workspace
+
+# install common dev requirements
+COPY requirements/dev.txt /tmp/requirements.txt
+RUN python -m pip install -r /tmp/requirements.txt
+
+# -----------------------------------------------------------------------------
+# Install Merlin Repos
+# -----------------------------------------------------------------------------
+
+COPY --chown=$USER merlin merlin
+
+# # # install merlin packages in development mode
+RUN python -m pip install -e merlin/models --no-deps
+RUN python -m pip install -e merlin/core --no-deps
+RUN python -m pip install -e merlin/NVTabular --no-deps
+RUN python -m pip install -e merlin/systems --no-deps
+RUN python -m pip install -e merlin/dataloader --no-deps

docker/requirements.Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.8.16-slim-buster
+RUN apt-get update && apt-get install -y git
+RUN pip install --upgrade pip && pip install pip-tools==6.12.3

requirements/compile.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker compose run --rm requirements sh -c "pip-compile --output-file=$1.txt --resolver=backtracking $1.in"

requirements/dev.cpu.in

@@ -0,0 +1,9 @@
+-r merlin-base.in
+-r merlin-dev.in
+
+# conflict between TensorFlow and cudf
+protobuf==3.20.3
+
+scipy
+tensorflow
+torch