Skip to content

Commit

Permalink
Merge branch 'master' into mateuszm/slicescatter/cpu
Browse files Browse the repository at this point in the history
  • Loading branch information
mlukasze authored Sep 16, 2024
2 parents 96a5ee4 + ace08e9 commit ee59ddd
Show file tree
Hide file tree
Showing 114 changed files with 27,252 additions and 45,343 deletions.
2 changes: 1 addition & 1 deletion .github/dockerfiles/docker_tag
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pr-26336
pr-26588
2 changes: 2 additions & 0 deletions .github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ RUN apt-get update && \
python3.11-venv \
python3.11-distutils \
libhdf5-dev \
# For TF Models tests
wget \
&& \
rm -rf /var/lib/apt/lists/*

Expand Down
9 changes: 7 additions & 2 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@
- 'tests/layer_tests/tensorflow2_keras_tests/**/*'
- 'tests/layer_tests/jax_tests/**/*'
- any: ['tests/model_hub_tests/**',
'!tests/model_hub_tests/pytorch/**/*']
'!tests/model_hub_tests/pytorch/**/*',
'!tests/model_hub_tests/jax/**/*']

'category: TFL FE':
- 'src/frontends/tensorflow_lite/**/*'
Expand All @@ -165,12 +166,16 @@
- 'tests/layer_tests/py_frontend_tests/test_torch_decoder.py'
- 'tests/layer_tests/py_frontend_tests/test_torch_frontend.py'
- any: ['tests/model_hub_tests/**',
'!tests/model_hub_tests/tensorflow/**/*']
'!tests/model_hub_tests/tensorflow/**/*',
'!tests/model_hub_tests/jax/**/*']

'category: JAX FE':
- 'src/frontends/jax/**/*'
- 'src/bindings/python/src/openvino/frontend/jax/**/*'
- 'tests/layer_tests/jax_tests/**/*'
- any: ['tests/model_hub_tests/**',
'!tests/model_hub_tests/tensorflow/**/*',
'!tests/model_hub_tests/pytorch/**/*']

'category: tools':
- any: ['tools/**',
Expand Down
14 changes: 13 additions & 1 deletion .github/workflows/android_x64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ concurrency:

permissions: read-all

env:
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}

jobs:
Smart_CI:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -84,6 +87,7 @@ jobs:
SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt
SCCACHE_LOG: warn
OPENVINO_REPO: '/__w/openvino/openvino/openvino'
OPENVINO_GENAI_REPO: '/__w/openvino/openvino/openvino.genai'
BUILD_DIR: '/__w/openvino/openvino/build'
ANDROID_TOOLS: '/deps/android_tools'
ANDROID_NDK_HOME: '/deps/android_tools/ndk-bundle'
Expand All @@ -98,6 +102,14 @@ jobs:
path: 'openvino'
submodules: 'true'

- name: Clone OpenVINO GenAI
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: 'openvinotoolkit/openvino.genai'
path: ${{ env.OPENVINO_GENAI_REPO }}
submodules: 'true'
ref: ${{ env.TARGET_BRANCH }}

#
# Print system info
#
Expand All @@ -117,13 +129,13 @@ jobs:
-DANDROID_STL=c++_shared \
-DANDROID_PLATFORM=${{ env.ANDROID_SDK_VERSION }} \
-DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DENABLE_STRICT_DEPENDENCIES=OFF \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \
-DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
-DENABLE_LTO=ON \
-DENABLE_PYTHON=OFF \
-DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_GENAI_REPO }} \
-S ${OPENVINO_REPO} \
-B ${BUILD_DIR}
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/build_doc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ jobs:
tar -xzf doxygen-$DOXY_VER.linux.bin.tar.gz
echo "$(pwd)/doxygen-$DOXY_VER/bin/" >> $GITHUB_PATH
- name: Validate benchmarks files
run: |
python3 docs/scripts/tests/validate_benchmarks.py docs/sphinx_setup/_static/benchmarks_files/
- name: Build docs
run: |
rm -rf build && mkdir build && cd build
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/coverity.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ on:
openvinoRef:
description: 'Branch, tag or commit hash to clone openvino from. Taken from event context if not set'
type: string
openvinoContribRef:
description: 'Branch, tag or commit hash to clone openvino_contrib from. Taken from event context if not set'
type: string
schedule:
# run daily at 00:00
- cron: '0 0 * * *'
Expand All @@ -19,6 +22,7 @@ permissions: read-all
env:
PIP_CACHE_PATH: /mount/caches/pip/linux
PYTHON_VERSION: '3.11'
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}

jobs:
Build:
Expand Down Expand Up @@ -58,7 +62,7 @@ jobs:
repository: 'openvinotoolkit/openvino_contrib'
path: ${{ env.OPENVINO_CONTRIB_REPO }}
submodules: 'true'
ref: 'master'
ref: ${{ inputs.openvinoContribRef || env.TARGET_BRANCH }}

#
# Dependencies
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/job_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ permissions: read-all

env:
PIP_CACHE_PATH: /mount/caches/pip/linux
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}

jobs:
Build:
Expand Down Expand Up @@ -110,7 +111,7 @@ jobs:
repository: 'openvinotoolkit/openvino_contrib'
path: ${{ env.OPENVINO_CONTRIB_REPO }}
submodules: 'true'
ref: 'master'
ref: ${{ env.TARGET_BRANCH }}

- name: Generate product manifest and set CI_BUILD_NUMBER & CI_BUILD_DEV_TAG
id: create_manifest
Expand Down Expand Up @@ -282,7 +283,7 @@ jobs:

- name: Upload openvino RPM packages
if: ${{ inputs.build-rpm-packages }}
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: openvino_rpm_packages
path: ${{ env.BUILD_DIR }}/*.rpm
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/job_build_windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ permissions: read-all
env:
PIP_CACHE_PATH: /mount/caches/pip/win
PYTHON_VERSION: '3.11'
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}

jobs:
Build:
Expand Down Expand Up @@ -56,7 +57,7 @@ jobs:
with:
repository: 'openvinotoolkit/openvino_contrib'
path: 'openvino_contrib'
ref: 'master'
ref: ${{ env.TARGET_BRANCH }}

- name: Set CCACHE_REMOTE_DIR based on event
shell: bash
Expand Down
116 changes: 116 additions & 0 deletions .github/workflows/job_jax_models_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
name: JAX/Flax Models tests

on:
workflow_call:
inputs:
runner:
description: 'Machine on which the tests would run'
type: string
required: true
container:
description: 'JSON to be converted to the value of the "container" configuration for the job'
type: string
required: false
default: '{"image": null}'
model_scope:
description: 'Scope of models for testing.'
type: string
required: true

permissions: read-all

jobs:
JAX_Models_Tests:
name: JAX/Flax Models tests
timeout-minutes: ${{ inputs.model_scope == 'precommit' && 35 || 35 }}
runs-on: ${{ inputs.runner }}
container: ${{ fromJSON(inputs.container) }}
defaults:
run:
shell: bash
env:
DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
OPENVINO_REPO: ${{ github.workspace }}/openvino
INSTALL_DIR: ${{ github.workspace }}/install
INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
steps:
- name: Download OpenVINO package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_package
path: ${{ env.INSTALL_DIR }}

- name: Download OpenVINO tokenizers extension
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_tokenizers_wheel
path: ${{ env.INSTALL_DIR }}

- name: Download OpenVINO tests package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_tests
path: ${{ env.INSTALL_TEST_DIR }}

# Needed as ${{ github.workspace }} is not working correctly when using Docker
- name: Setup Variables
run: |
echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV"
echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV"
echo "MODEL_HUB_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/model_hub_tests" >> "$GITHUB_ENV"
- name: Extract OpenVINO packages
run: |
pushd ${INSTALL_DIR}
tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
popd
pushd ${INSTALL_TEST_DIR}
tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
popd
- name: Fetch setup_python action
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
sparse-checkout: |
.github/actions/setup_python/action.yml
sparse-checkout-cone-mode: false
path: 'openvino'

- name: Setup Python 3.11
uses: ./openvino/.github/actions/setup_python
with:
version: '3.11'
should-setup-pip-paths: 'false'
self-hosted-runner: ${{ contains(inputs.runner, 'aks') }}

- name: Install OpenVINO Python wheels
run: |
# To enable pytest parallel features
python3 -m pip install pytest-xdist[psutil]
python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-*
- name: Install JAX tests requirements for precommit
run: |
python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/jax/requirements.txt
- name: JAX/Flax Models Tests from Hugging Face
if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly' }}
run: |
export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/jax/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-jax_model_${{ inputs.model_scope }}_tests.html --self-contained-html -v
env:
TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }}
TEST_DEVICE: CPU

- name: Upload Test Results
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
if: ${{ !cancelled() }}
with:
name: test-results-jax-models-${{ inputs.model_scope }}
path: |
${{ env.INSTALL_TEST_DIR }}/TEST*.html
if-no-files-found: 'error'
17 changes: 10 additions & 7 deletions .github/workflows/job_python_unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,15 @@ on:
description: 'Components that are affected by changes in the commit defined by the Smart CI Action'
type: string
required: true
python-version:
description: 'Python version to setup. E.g., "3.11"'
type: string
required: true

permissions: read-all

env:
PIP_CACHE_PATH: /mount/caches/pip/linux
PYTHON_VERSION: '3.11'

jobs:
Python_Unit_Tests:
Expand Down Expand Up @@ -77,10 +80,10 @@ jobs:
sparse-checkout-cone-mode: false
path: 'action_root'

- name: Setup Python ${{ env.PYTHON_VERSION }}
- name: Setup Python ${{ inputs.python-version }}
uses: ./action_root/.github/actions/setup_python
with:
version: ${{ env.PYTHON_VERSION }}
version: ${{ inputs.python-version }}
pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }}
should-setup-pip-paths: ${{ runner.os == 'Linux' }}
self-hosted-runner: ${{ runner.os == 'Linux' }}
Expand Down Expand Up @@ -174,7 +177,7 @@ jobs:
export LD_PRELOAD=${GOMP_LIB}
fi
python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/mo_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_mo_convert.xml
python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/mo_python_api_tests -n logical --junitxml=${INSTALL_TEST_DIR}/TEST-test_mo_convert.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
Expand All @@ -192,7 +195,7 @@ jobs:
export LD_PRELOAD=${GOMP_LIB}
fi
python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/ovc_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_ovc_convert.xml
python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/ovc_python_api_tests -n logical --junitxml=${INSTALL_TEST_DIR}/TEST-test_ovc_convert.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
Expand Down Expand Up @@ -231,7 +234,7 @@ jobs:
PYTORCH_TRACING_MODE: TORCHFX

- name: ONNX Layer Tests
if: fromJSON(inputs.affected-components).ONNX_FE.test
if: ${{ fromJSON(inputs.affected-components).ONNX_FE.test }}
run: |
# requires 'unit_tests' from 'tools/mo'
export PYTHONPATH=${INSTALL_TEST_DIR}/mo:$PYTHONPATH
Expand Down Expand Up @@ -296,7 +299,7 @@ jobs:
python3 ${OPENVINO_REPO}/docs/articles_en/assets/snippets/main.py
- name: Python API Tests -- numpy>=2.0.0
if: ${{ fromJSON(inputs.affected-components).Python_API.test }}
if: ${{ fromJSON(inputs.affected-components).Python_API.test && inputs.python-version != '3.12' }} # Ticket: 152242
run: |
python3 -m pip uninstall -y numpy
python3 -m pip install "numpy>=2.0.0,<2.1.0"
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/job_tokenizers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ permissions: read-all
env:
PIP_CACHE_PATH: /mount/caches/pip/linux
PYTHON_VERSION: '3.11'
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}

jobs:
Openvino_tokenizers:
Expand Down Expand Up @@ -70,7 +71,7 @@ jobs:
with:
repository: 'openvinotoolkit/openvino_tokenizers'
path: ${{ env.OPENVINO_TOKENIZERS_REPO }}
ref: 'master'
ref: ${{ env.TARGET_BRANCH }}

- name: Download OpenVINO package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/linux_arm64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ permissions: read-all

env:
PIP_CACHE_PATH: /mount/caches/pip/linux
PYTHON_VERSION: '3.11'
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}

jobs:
Smart_CI:
Expand Down Expand Up @@ -122,7 +122,7 @@ jobs:
repository: 'openvinotoolkit/openvino_contrib'
path: ${{ env.OPENVINO_CONTRIB_REPO }}
submodules: 'true'
ref: 'master'
ref: ${{ env.TARGET_BRANCH }}

- name: Generate product manifest and set CI_BUILD_NUMBER & CI_BUILD_DEV_TAG
id: create_manifest
Expand Down Expand Up @@ -388,6 +388,7 @@ jobs:
runner: 'aks-linux-16-cores-arm'
container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_arm64 }}", "volumes": ["/mount:/mount"]}'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
python-version: '3.11'

TensorFlow_Layer_Tests:
name: TensorFlow Layer Tests
Expand Down
Loading

0 comments on commit ee59ddd

Please sign in to comment.