add PR CI for cugraph-pyg and cugraph-dgl

.github/workflows/pr.yaml

@@ -20,6 +20,10 @@ jobs:
       - conda-python-tests
       - wheel-build-pylibwholegraph
       - wheel-tests-pylibwholegraph
+      - wheel-build-cugraph-dgl
+      - wheel-tests-cugraph-dgl
+      - wheel-build-cugraph-pyg
+      - wheel-tests-cugraph-pyg
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12
     if: always()
@@ -104,3 +108,35 @@ jobs:
       build_type: pull-request
       script: ci/test_wheel_pylibwholegraph.sh
       matrix_filter: map(select(.ARCH == "amd64"))
+  wheel-build-cugraph-dgl:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_cugraph-dgl.sh
+  wheel-tests-cugraph-dgl:
+    needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-dgl, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_cugraph-dgl.sh
+      matrix_filter: map(select(.ARCH == "amd64"))
+  wheel-build-cugraph-pyg:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_cugraph-pyg.sh
+  wheel-tests-cugraph-pyg:
+    needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-pyg, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_cugraph-pyg.sh
+      matrix_filter: map(select(.ARCH == "amd64"))

.gitignore

@@ -70,9 +70,10 @@ cpp/thirdparty/googletest/
 *.iws
 
 ## Datasets
-datasets/*
+datasets/
 !datasets/cyber.csv
 !datasets/get_test_data.sh
+!datasets/karate.csv
 !datasets/karate-data.csv
 !datasets/karate_undirected.csv
 !datasets/karate-disjoint.csv

.pre-commit-config.yaml

@@ -54,7 +54,7 @@ repos:
               setup[.]cfg$
       - id: verify-alpha-spec
   - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.15.1
+    rev: v1.16.0
     hooks:
         - id: rapids-dependency-file-generator
           args: ["--clean"]

ci/build_cpp.sh

@@ -17,6 +17,10 @@ version=$(rapids-generate-version)
 
 rapids-logger "Begin cpp build"
 
+sccache --zero-stats
+
 RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libwholegraph
 
+sccache --show-adv-stats
+
 rapids-upload-conda-to-s3 cpp

ci/build_python.sh

@@ -17,6 +17,8 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 rapids-generate-version > ./VERSION
 
+sccache --zero-stats
+
 # TODO: Remove `--no-test` flags once importing on a CPU
 # node works correctly
 rapids-logger "Begin pylibwholegraph build"
@@ -25,4 +27,16 @@ RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/pylibwholegraph
 
+sccache --show-adv-stats
+
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  conda/recipes/cugraph-pyg
+
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  conda/recipes/cugraph-dgl
+
 rapids-upload-conda-to-s3 python

ci/build_wheel.sh

@@ -32,6 +32,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 cd "${package_dir}"
 
+sccache --zero-stats
+
 rapids-logger "Building '${package_name}' wheel"
 python -m pip wheel \
     -w dist \

ci/run_cugraph_dgl_pytests.sh

@@ -4,6 +4,6 @@
 set -euo pipefail
 
 # Support invoking run_cugraph_dgl_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/cugraph_dgl
 
 pytest --cache-clear --ignore=mg "$@" .

ci/run_cugraph_pyg_pytests.sh

@@ -9,7 +9,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-pyg/cugraph_
 pytest --cache-clear --benchmark-disable "$@" .
 
 # Used to skip certain examples in CI due to memory limitations
-export CI_RUN=1
+export CI=true
 
 # Test examples
 for e in "$(pwd)"/examples/*.py; do

ci/test_wheel_cugraph-dgl.sh

@@ -4,24 +4,15 @@
 set -eoxu pipefail
 
 package_name="cugraph-dgl"
-package_dir="python/cugraph-dgl"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
-RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
+# Download the pylibwholegraph and cugraph-dgl built in the previous step
+RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
 
+# determine pytorch and DGL sources
 PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
 PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
 if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
@@ -30,20 +21,19 @@ else
   PYTORCH_CUDA_VER=$PKG_CUDA_VER
 fi
 PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}"
-DGL_URL="https://data.dgl.ai/wheels/cu${PYTORCH_CUDA_VER}/repo.html"
-
-# Starting from 2.2, PyTorch wheels depend on nvidia-nccl-cuxx>=2.19 wheel and
-# dynamically link to NCCL. RAPIDS CUDA 11 CI images have an older NCCL version that
-# might shadow the newer NCCL required by PyTorch during import (when importing
-# `cupy` before `torch`).
-if [[ "${NCCL_VERSION}" < "2.19" ]]; then
-  PYTORCH_VER="2.1.0"
-else
-  PYTORCH_VER="2.3.0"
-fi
-
-rapids-logger "Installing PyTorch and DGL"
-rapids-retry python -m pip install "torch==${PYTORCH_VER}" --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install dgl==2.0.0 --find-links ${DGL_URL}
-
-python -m pytest python/cugraph-dgl/tests
+DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html"
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${DGL_URL}" \
+    "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+    'dgl==2.4.0' \
+    'torch>=2.0,<2.4.0a0'
+
+# RAPIDS_DATASET_ROOT_DIR is used by test scripts
+export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
+
+python -m pytest python/cugraph-dgl/cugraph_dgl/tests

ci/test_wheel_cugraph-pyg.sh

@@ -4,46 +4,33 @@
 set -eoxu pipefail
 
 package_name="cugraph-pyg"
-package_dir="python/cugraph-pyg"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
-RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
+# Download the pylibwholegraph and cugraph-pyg built in the previous step
+RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
-# RAPIDS_DATASET_ROOT_DIR is used by test scripts
-export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
-
-# Used to skip certain examples in CI due to memory limitations
-export CI_RUN=1
 
+# determine pytorch and pyg sources
 if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then
   PYTORCH_URL="https://download.pytorch.org/whl/cu118"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html"
 else
   PYTORCH_URL="https://download.pytorch.org/whl/cu121"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html"
 fi
-rapids-logger "Installing PyTorch and PyG dependencies"
-rapids-retry python -m pip install torch==2.1.0 --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install "torch-geometric>=2.5,<2.6"
-rapids-retry python -m pip install \
-  ogb \
-  pyg_lib \
-  torch_scatter \
-  torch_sparse \
-  tensordict \
-  -f ${PYG_URL}
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${PYG_URL}" \
+    "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
+
+# RAPIDS_DATASET_ROOT_DIR is used by test scripts
+export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
 
 rapids-logger "pytest cugraph-pyg (single GPU)"
 pushd python/cugraph-pyg/cugraph_pyg

ci/test_wheel_pylibwholegraph.sh

@@ -25,13 +25,8 @@ mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 rapids-logger "Installing Packages"
 rapids-retry python -m pip install \
     --extra-index-url ${INDEX_URL} \
-    "$(echo ./dist/pylibwholegraph*.whl)[test]"
-
-# install torch separately, to be sure we get a CUDA build
-python -m pip install \
-  --index-url "${INDEX_URL}" \
-  -v \
-  'torch>=2.0,<2.4.0a0'
+    "$(echo ./dist/pylibwholegraph*.whl)[test]" \
+    'torch>=2.0,<2.4.0a0'
 
 rapids-logger "pytest pylibwholegraph"
 cd python/pylibwholegraph/pylibwholegraph/tests

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -4,7 +4,6 @@ channels:
 - rapidsai
 - rapidsai-nightly
 - dask/label/dev
-- pyg
 - conda-forge
 - nvidia
 dependencies:
@@ -18,7 +17,7 @@ dependencies:
 - cupy>=12.0.0
 - cython>=3.0.0
 - dask-cudf==24.12.*,>=0.0.0a0
-- dglteam/label/th21_cu118::dgl
+- dglteam/label/th23_cu118::dgl>=2.4.0.th23.cu*
 - doxygen
 - graphviz
 - ipython
@@ -33,7 +32,6 @@ dependencies:
 - pre-commit
 - pydantic
 - pydata-sphinx-theme
-- pyg::pyg
 - pylibcugraphops==24.12.*,>=0.0.0a0
 - pylibraft==24.12.*,>=0.0.0a0
 - pytest
@@ -42,16 +40,20 @@ dependencies:
 - pytest-forked
 - pytest-xdist
 - pytorch-cuda=11.8
-- pytorch::pytorch>=2.0,<2.4.0a0
+- pytorch::pytorch>=2.3,<2.4.0a0
+- pytorch_geometric>=2.5,<2.6
 - raft-dask==24.12.*,>=0.0.0a0
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
 - rmm==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - scipy
+- setuptools>=61.0.0
 - sphinx-copybutton
 - sphinx-markdown-tables
 - sphinx<6
 - sphinxcontrib-websupport
 - torchdata
 - wget
+- wheel
 name: all_cuda-118_arch-x86_64

conda/environments/all_cuda-121_arch-x86_64.yaml

@@ -4,7 +4,6 @@ channels:
 - rapidsai
 - rapidsai-nightly
 - dask/label/dev
-- pyg
 - conda-forge
 - nvidia
 dependencies:
@@ -19,7 +18,7 @@ dependencies:
 - cupy>=12.0.0
 - cython>=3.0.0
 - dask-cudf==24.12.*,>=0.0.0a0
-- dglteam/label/th21_cu121::dgl
+- dglteam/label/th23_cu121::dgl>=2.4.0.th23.cu*
 - doxygen
 - graphviz
 - ipython
@@ -38,7 +37,6 @@ dependencies:
 - pre-commit
 - pydantic
 - pydata-sphinx-theme
-- pyg::pyg
 - pylibcugraphops==24.12.*,>=0.0.0a0
 - pylibraft==24.12.*,>=0.0.0a0
 - pytest
@@ -47,16 +45,20 @@ dependencies:
 - pytest-forked
 - pytest-xdist
 - pytorch-cuda=12.1
-- pytorch::pytorch>=2.0,<2.4.0a0
+- pytorch::pytorch>=2.3,<2.4.0a0
+- pytorch_geometric>=2.5,<2.6
 - raft-dask==24.12.*,>=0.0.0a0
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
 - rmm==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - scipy
+- setuptools>=61.0.0
 - sphinx-copybutton
 - sphinx-markdown-tables
 - sphinx<6
 - sphinxcontrib-websupport
 - torchdata
 - wget
+- wheel
 name: all_cuda-121_arch-x86_64