test(pu&nyz): add self-hosted linux runner for ci (#323)

* fix(pu): use macos-13 in action

* fix(pu): use unittest

* fix(pu): fix unittest

* fix(pu): fix unittest

* fix(pu): fix test.yml

.github/workflows/test.yml

@@ -26,15 +26,26 @@ jobs:
           - '3.9'
           - '3.10'
     steps:
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Get system version for Linux
         if: ${{ runner.os == 'Linux' }}
         shell: bash
         run: |
-          sudo apt-get update -y
-          sudo apt-get install swig -y
+          sudo dpkg --configure -a
+          sudo apt update && sudo apt upgrade -y
+          sudo apt-get install -y build-essential clang libboost-python-dev libeigen3-dev
+          sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libopencv-dev swig tzdata
+          sudo apt-get install -y python${{ matrix.python-version }}-dev
+          python3-config --includes
           echo "OS_NAME=Linux" >> $GITHUB_ENV
           echo "IS_WIN=" >> $GITHUB_ENV
           echo "IS_MAC=" >> $GITHUB_ENV
+          echo "CFLAGS=$(python3-config --includes)" >> $GITHUB_ENV
+          echo "LDFLAGS=$(python3-config --ldflags)" >> $GITHUB_ENV
       - name: Get system version for MacOS
         if: ${{ runner.os == 'macOS' }}
         shell: bash
@@ -46,33 +57,40 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 20
-      - name: Set up system dependences on MacOS
+      - name: Set up system dependencies on MacOS
         if: ${{ env.OS_NAME == 'MacOS' }}
         shell: bash
         run: |
           brew install tree cloc wget curl make zip graphviz
           brew install llvm  # Install llvm (which includes clang)
+          brew install opencv  # Install OpenCV
           echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> $GITHUB_ENV  # update PATH
           dot -V
       - name: Set CC and CXX variables
         run: |
           echo "CC=$(which clang)" >> $GITHUB_ENV
           echo "CXX=$(which clang++)" >> $GITHUB_ENV
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+#          echo "CC=$(which gcc)" >> $GITHUB_ENV
+#          echo "CXX=$(which g++)" >> $GITHUB_ENV
       - name: Install PyPI dependencies
         shell: bash
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade pip setuptools wheel cython
           pip install --no-cache-dir --upgrade flake8 setuptools wheel twine -i https://pypi.tuna.tsinghua.edu.cn/simple
           pip install -r requirements.txt --upgrade --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple
           pip install -r requirements-build.txt --upgrade --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple
           pip install -r requirements-test.txt --upgrade --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple
           pip uninstall DI-engine -y
-          pip install --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple git+http://github.com/opendilab/DI-engine.git@main#egg=DI-engine[common_env] --upgrade
+          pip install --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple git+https://github.com/opendilab/DI-engine.git@main#egg=DI-engine[common_env] --upgrade
           pip install --no-cache-dir transformers --upgrade -i https://pypi.tuna.tsinghua.edu.cn/simple
+          pip install opencv-python --upgrade --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple
+      - name: Verify Python.h availability
+        shell: bash
+        run: |
+          if [ ! -f "$(python3-config --includes | sed 's/-I//g' | awk '{print $1}')/Python.h" ]; then
+            echo "Error: Python.h not found."
+            exit 1
+          fi
       - name: Run unittest on MacOS or Linux
         if: ${{ env.OS_NAME == 'MacOS' || env.OS_NAME == 'Linux'}}
         env:
@@ -82,7 +100,7 @@ jobs:
         run: |
             make clean build unittest
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.xml

lzero/mcts/buffer/game_buffer.py

@@ -549,7 +549,12 @@ def _push_game_segment(self, data: Any, meta: Optional[dict] = None) -> None:
         Returns:
             - buffered_data (:obj:`BufferedData`): The pushed data.
         """
-        data_length = len(data.action_segment) if len(data.action_segment)<self._cfg.game_segment_length else self._cfg.game_segment_length
+        try:
+            data_length = len(data.action_segment) if len(data.action_segment) < self._cfg.game_segment_length else self._cfg.game_segment_length
+        except Exception as e:
+            # to be compatible with unittest
+            print(e)
+            data_length = len(data)
 
         if meta['done']:
             self.num_of_collected_episodes += 1

lzero/mcts/tests/test_game_buffer.py

@@ -15,6 +15,7 @@
         env_type='not_board_games',
         use_priority=True,
         action_type='fixed_action_space',
+        game_segment_length=20,
     )
 )
 

lzero/mcts/tests/test_muzero_game_buffer.py

@@ -3,7 +3,7 @@
 import torch
 from ding.config import compile_config
 from ding.policy import create_policy
-from huggingface_hub import hf_hub_url, cached_download
+from huggingface_hub import hf_hub_download
 
 from lzero.mcts.buffer.game_buffer_efficientzero import MuZeroGameBuffer
 from lzero.model.muzero_model import MuZeroModel as Model
@@ -40,11 +40,9 @@
 # initialize replay buffer
 replay_buffer = MuZeroGameBuffer(cfg.policy)
 
-# get the download link of the test data from Hugging Face
-url = hf_hub_url("puyuan1996/pong_muzero_2episodes_gsl400_v0.0.4", "pong_muzero_2episodes_gsl400_v0.0.4.npy",
-                 repo_type='dataset')
 # download and cache the file
-local_filepath = cached_download(url)
+local_filepath = hf_hub_download("puyuan1996/pong_muzero_2episodes_gsl400_v0.0.4", "pong_muzero_2episodes_gsl400_v0.0.4.npy", repo_type='dataset')
+
 # load .npy file
 data = np.load(local_filepath, allow_pickle=True)
 

requirements-test.txt

@@ -10,7 +10,7 @@ pytest-timeout~=2.0.2
 pytest-benchmark~=3.4.0
 testtools>=2
 hbutils>=0.6.13
-setuptools<=59.5.0
+setuptools>=66.1.1
 numpy>=1.21.6
 easydict>=1.7,<2
 swig >= 4.1.1

requirements.txt

@@ -1,6 +1,6 @@
-DI-engine>=0.4.7
+DI-engine>=0.5.3
 gymnasium[atari]==0.28.0
-numpy==1.24.1
+numpy>=1.24.1,<2
 pympler
 minigrid
 moviepy

setup.py

@@ -20,6 +20,18 @@
 
 here = os.path.abspath(os.path.dirname(__file__))
 
+from distutils.sysconfig import get_python_inc
+
+python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+python_include_dir = get_python_inc()
+include_dirs = [np.get_include(), python_include_dir]
+if sys.platform == 'darwin':
+    homebrew_python_path = f'/usr/local/opt/python@{python_version}/Frameworks/Python.framework/Versions/{python_version}/include/python{python_version}'
+    if os.path.exists(homebrew_python_path):
+        include_dirs.append(homebrew_python_path)
+print(f"Python version: {python_version}")
+print(f"Include directories: {include_dirs}")
+
 
 def _load_req(file: str):
     with open(file, 'r', encoding='utf-8') as f:
@@ -72,7 +84,8 @@ def find_cython_extensions(path=None):
         extname = '.'.join(rpath.split(os.path.sep))
         extensions.append(Extension(
             extname, [item],
-            include_dirs=[np.get_include()],
+            # include_dirs=[np.get_include()],
+            include_dirs=include_dirs,
             language="c++",
             extra_compile_args=extra_compile_args,
             extra_link_args=extra_link_args,