reverted python code. #513
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Python Package using Conda | ||
on: | ||
pull_request: | ||
branches: [ main, dev ] | ||
push: | ||
jobs: | ||
build-and-test: | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
os: [ ubuntu-20.04, ubuntu-22.04 ] | ||
profiler: [ DEFAULT, DLIO_PROFILER ] | ||
gcc: [10] | ||
python: ["3.8", "3.9", "3.10"] | ||
name: ${{ matrix.os }}-${{ matrix.profiler }}-${{ matrix.gcc }}-${{ matrix.python }} | ||
runs-on: ${{ matrix.os }} | ||
env: | ||
VENV: "/home/runner/work/venv" | ||
DLIO_PROFILER: ${{ matrix.profiler }} | ||
CC: gcc-${{ matrix.gcc }} | ||
CXX: g++-${{ matrix.gcc }} | ||
RDMAV_FORK_SAFE: "1" | ||
DLIO_PROFILER_LOG_LEVEL: "INFO" | ||
PYTHON_VER: ${{ matrix.python }} | ||
steps: | ||
- name: Push checkout | ||
if: github.event_name == 'push' | ||
uses: actions/checkout@v3 | ||
- name: PR checkout | ||
if: github.event_name == 'pull_request' | ||
uses: actions/checkout@v3 | ||
with: | ||
ref: ${{ github.event.pull_request.head.sha }} | ||
name: Set up Python ${{ matrix.python }} | ||
uses: actions/setup-python@v3 | ||
with: | ||
python-version: ${{ matrix.python }} | ||
- name: Cache install modules | ||
id: cache-modules | ||
uses: actions/cache@v3 | ||
with: | ||
path: ${{ env.VENV }} | ||
key: ${{env.VENV }}-${{env.DLIO_PROFILER}}-${{ matrix.gcc }}-${{ matrix.python }}-${{ hashFiles('setup.py') }} | ||
- name: Install System Tools | ||
run: | | ||
sudo apt update | ||
sudo apt-get install $CC $CXX libc6 | ||
sudo apt-get install mpich | ||
if [[ $DLIO_PROFILER == 'DLIO_PROFILER' ]]; then | ||
sudo apt-get install libhwloc-dev | ||
fi | ||
- name: Install DLIO | ||
if: steps.cache-modules.outputs.cache-hit != 'true' | ||
run: | | ||
echo "Profiler ${DLIO_PROFILER} gcc $CC" | ||
python -m pip install --upgrade pip | ||
pip install virtualenv | ||
python -m venv ${VENV} | ||
source ${VENV}/bin/activate | ||
if [[ $DLIO_PROFILER == 'DLIO_PROFILER' ]]; then | ||
pip install .[test,dlio_profiler] | ||
else | ||
pip install .[test] | ||
fi | ||
rm -rf dlio_benchmark | ||
- name: test_gen_data | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_gen_data[png-tensorflow] -v | ||
mpirun -np 2 pytest -k test_gen_data[npz-tensorflow] -v | ||
mpirun -np 2 pytest -k test_gen_data[jpeg-tensorflow] -v | ||
mpirun -np 2 pytest -k test_gen_data[tfrecord-tensorflow] -v | ||
mpirun -np 2 pytest -k test_gen_data[hdf5-tensorflow] -v | ||
- name: test_custom_storage_root_gen_data | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_storage_root_gen_data[png-tensorflow] -v | ||
mpirun -np 2 pytest -k test_storage_root_gen_data[npz-tensorflow] -v | ||
mpirun -np 2 pytest -k test_storage_root_gen_data[jpeg-tensorflow] -v | ||
mpirun -np 2 pytest -k test_storage_root_gen_data[tfrecord-tensorflow] -v | ||
mpirun -np 2 pytest -k test_storage_root_gen_data[hdf5-tensorflow] -v | ||
- name: test_train | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_train[png-tensorflow-tensorflow] -v | ||
mpirun -np 2 pytest -k test_train[npz-tensorflow-tensorflow] -v | ||
mpirun -np 2 pytest -k test_train[jpeg-tensorflow-tensorflow] -v | ||
mpirun -np 2 pytest -k test_train[tfrecord-tensorflow-tensorflow] -v | ||
mpirun -np 2 pytest -k test_train[hdf5-tensorflow-tensorflow] -v | ||
mpirun -np 2 pytest -k test_train[csv-tensorflow-tensorflow] -v | ||
mpirun -np 2 pytest -k test_train[png-pytorch-pytorch] -v | ||
mpirun -np 2 pytest -k test_train[npz-pytorch-pytorch] -v | ||
mpirun -np 2 pytest -k test_train[jpeg-pytorch-pytorch] -v | ||
mpirun -np 2 pytest -k test_train[hdf5-pytorch-pytorch] -v | ||
mpirun -np 2 pytest -k test_train[csv-pytorch-pytorch] -v | ||
mpirun -np 2 pytest -k test_train[png-tensorflow-dali] -v | ||
mpirun -np 2 pytest -k test_train[npz-tensorflow-dali] -v | ||
mpirun -np 2 pytest -k test_train[jpeg-tensorflow-dali] -v | ||
mpirun -np 2 pytest -k test_train[hdf5-tensorflow-dali] -v | ||
mpirun -np 2 pytest -k test_train[csv-tensorflow-dali] -v | ||
mpirun -np 2 pytest -k test_train[png-pytorch-dali] -v | ||
mpirun -np 2 pytest -k test_train[npz-pytorch-dali] -v | ||
mpirun -np 2 pytest -k test_train[jpeg-pytorch-dali] -v | ||
mpirun -np 2 pytest -k test_train[hdf5-pytorch-dali] -v | ||
mpirun -np 2 pytest -k test_train[csv-pytorch-dali] -v | ||
- name: test_custom_storage_root_train | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[png-tensorflow] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[npz-tensorflow] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[jpeg-tensorflow] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[tfrecord-tensorflow] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[hdf5-tensorflow] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[csv-tensorflow] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[png-pytorch] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[npz-pytorch] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[jpeg-pytorch] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[hdf5-pytorch] -v | ||
mpirun -np 2 pytest -k test_custom_storage_root_train[csv-pytorch] -v | ||
- name: test_checkpoint_epoch | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_checkpoint_epoch -v | ||
- name: test_checkpoint_step | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_checkpoint_step -v | ||
- name: test_eval | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_eval -v | ||
- name: test_multi_threads | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 pytest -k test_multi_threads[tensorflow-0] -v | ||
mpirun -np 2 pytest -k test_multi_threads[tensorflow-1] -v | ||
mpirun -np 2 pytest -k test_multi_threads[tensorflow-2] -v | ||
mpirun -np 2 pytest -k test_multi_threads[pytorch-0] -v | ||
mpirun -np 2 pytest -k test_multi_threads[pytorch-1] -v | ||
mpirun -np 2 pytest -k test_multi_threads[pytorch-2] -v | ||
- name: test-tf-loader-tfrecord | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 dlio_benchmark workload=resnet50 ++workload.dataset.num_files_train=64 ++workload.workflow.train=False ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_samples_per_file=16 | ||
mpirun -np 2 dlio_benchmark workload=resnet50 ++workload.dataset.num_files_train=64 ++workload.workflow.train=True ++workload.workflow.generate_data=False ++workload.dataset.num_files_train=16 ++workload.dataset.num_samples_per_file=16 | ||
- name: test-torch-loader-npz | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 dlio_benchmark workload=unet3d ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=False ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 | ||
mpirun -np 2 dlio_benchmark workload=unet3d ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=False ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 | ||
- name: test-tf-loader-npz | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 dlio_benchmark workload=unet3d ++workload.framework=tensorflow ++workload.data_reader.data_loader=tensorflow ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=False ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 | ||
mpirun -np 2 dlio_benchmark workload=unet3d ++workload.framework=tensorflow ++workload.data_reader.data_loader=tensorflow ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=False ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 | ||
- name: test_subset | ||
run: | | ||
source ${VENV}/bin/activate | ||
mpirun -np 2 dlio_benchmark ++workload.workflow.generate_data=True ++workload.workflow.train=False | ||
mpirun -np 2 dlio_benchmark ++workload.workflow.generate_data=False ++workload.workflow.train=True ++workload.dataset.num_files_train=8 |