2025-02-22 nightly release (54b371f)

.ci/scripts/setup-linux.sh

@@ -22,7 +22,7 @@ fi
 # have already been installed, so we use PyTorch build from source here instead
 # of nightly. This allows CI to test against latest commits from PyTorch
 install_executorch "use-pt-pinned-commit"
-build_executorch_runner "${BUILD_TOOL}"
+build_executorch_runner "${BUILD_TOOL}" "${2:-Release}"
 
 if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then
   do_not_use_nightly_on_ci

.ci/scripts/setup-macos.sh

@@ -136,7 +136,7 @@ install_pytorch_and_domains
 # We build PyTorch from source here instead of using nightly. This allows CI to test against
 # the pinned commit from PyTorch
 install_executorch "use-pt-pinned-commit"
-build_executorch_runner "${BUILD_TOOL}"
+build_executorch_runner "${BUILD_TOOL}" "${2:-Release}"
 
 if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then
   do_not_use_nightly_on_ci

.ci/scripts/test_qnn_static_llama.sh

@@ -34,11 +34,11 @@ $PYTHON_EXECUTABLE -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o to
 
 set +e
 # Compile only as weight sharing is not applicable on x86
-$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleScript.test_stories_single_llama --model SM8650 --build_folder build-android/ --executorch_root . --artifact_dir . --compile_only
+$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleLLMScript.test_llama_stories_110m --model SM8650 --build_folder build-android/ --executorch_root . --artifact_dir . --llama_artifacts . --compile_only
 exit_code1=$?
 
 # Checks accuracy with weight sharing disabled since x86 does not support weight sharing.
-$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleScript.test_stories_single_llama --model SM8650 --build_folder build-x86/ --executorch_root . --artifact_dir . --enable_x86_64
+$PYTHON_EXECUTABLE backends/qualcomm/tests/test_qnn_delegate.py -k TestExampleLLMScript.test_llama_stories_110m --model SM8650 --build_folder build-x86/ --executorch_root . --artifact_dir . --llama_artifacts . --enable_x86_64
 exit_code2=$?
 
 # Check the exit codes and print messages

.ci/scripts/unittest-buck2.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# TODO: expand this to //...
+buck2 query //runtime/...
+
+# TODO: expand the covered scope of Buck targets.
+buck2 build //runtime/core/portable_type/...
+buck2 test //runtime/core/portable_type/...

.ci/scripts/unittest-linux-cmake.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# Run pytest with coverage
+pytest -n auto --cov=./ --cov-report=xml
+# Run gtest
+LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \
+test/run_oss_cpp_tests.sh

.ci/scripts/unittest-linux.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+BUILD_TOOL=$1
+if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then
+    echo "Running unittests for ${BUILD_TOOL} ..."
+else
+  echo "Missing build tool (require buck2 or cmake), exiting..."
+  exit 1
+fi
+
+BUILD_MODE=$2
+if [[ "${BUILD_MODE:-}" =~ ^(Debug|Release)$ ]]; then
+    echo "Running tests in build mode ${BUILD_MODE} ..."
+else
+    echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release."
+    exit 1
+fi
+
+# The generic Linux job chooses to use base env, not the one setup by the image
+eval "$(conda shell.bash hook)"
+CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+conda activate "${CONDA_ENV}"
+
+# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
+source .ci/scripts/setup-vulkan-linux-deps.sh
+
+PYTHON_EXECUTABLE=python \
+EXECUTORCH_BUILD_PYBIND=ON \
+CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
+.ci/scripts/setup-linux.sh "$BUILD_TOOL" "$BUILD_MODE"
+
+# Install llama3_2_vision dependencies.
+PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
+
+if [[ "$BUILD_TOOL" == "cmake" ]]; then
+    .ci/scripts/unittest-linux-cmake.sh
+elif [[ "$BUILD_TOOL" == "buck2" ]]; then
+    .ci/scripts/unittest-buck2.sh
+else
+    echo "Unknown build tool $BUILD_TOOL"
+    exit 1
+fi

.ci/scripts/unittest-macos-cmake.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# Run pytest with coverage
+${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml
+# Run gtest
+LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \
+${CONDA_RUN} test/run_oss_cpp_tests.sh

.ci/scripts/unittest-macos.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+BUILD_TOOL=$1
+if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then
+    echo "Running unittests for ${BUILD_TOOL} ..."
+else
+  echo "Missing build tool (require buck2 or cmake), exiting..."
+  exit 1
+fi
+
+BUILD_MODE=$2
+if [[ $BUILD_MODE =~ ^(Debug|Release)$ ]]; then
+    echo "Running tests in build mode ${BUILD_MODE} ..."
+else
+    echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release."
+    exit 1
+fi
+
+bash .ci/scripts/setup-conda.sh
+eval "$(conda shell.bash hook)"
+
+# Create temp directory for sccache shims
+export TMP_DIR=$(mktemp -d)
+export PATH="${TMP_DIR}:$PATH"
+trap 'rm -rfv ${TMP_DIR}' EXIT
+
+# Setup MacOS dependencies as there is no Docker support on MacOS atm
+PYTHON_EXECUTABLE=python \
+EXECUTORCH_BUILD_PYBIND=ON \
+CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
+${CONDA_RUN} --no-capture-output \
+.ci/scripts/setup-macos.sh "${BUILD_TOOL}" "${BUILD_MODE}"
+
+# Install llama3_2_vision dependencies.
+PYTHON_EXECUTABLE=python \
+${CONDA_RUN} --no-capture-output \
+./examples/models/llama3_2_vision/install_requirements.sh
+
+if [[ "$BUILD_TOOL" == "cmake" ]]; then
+    .ci/scripts/unittest-macos-cmake.sh
+elif [[ "$BUILD_TOOL" == "buck2" ]]; then
+    .ci/scripts/unittest-buck2.sh
+else
+    echo "Unknown build tool $BUILD_TOOL"
+    exit 1
+fi

.ci/scripts/utils.sh

@@ -109,7 +109,7 @@ build_executorch_runner_cmake() {
   pushd "${CMAKE_OUTPUT_DIR}" || return
   # This command uses buck2 to gather source files and buck2 could crash flakily
   # on MacOS
-  retry cmake -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" -DCMAKE_BUILD_TYPE=Release ..
+  retry cmake -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" -DCMAKE_BUILD_TYPE="${1:-Release}" ..
   popd || return
 
   if [ "$(uname)" == "Darwin" ]; then
@@ -124,7 +124,7 @@ build_executorch_runner() {
   if [[ $1 == "buck2" ]]; then
     build_executorch_runner_buck2
   elif [[ $1 == "cmake" ]]; then
-    build_executorch_runner_cmake
+    build_executorch_runner_cmake "$2"
   else
     echo "Invalid build tool $1. Only buck2 and cmake are supported atm"
     exit 1

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -21,7 +21,7 @@ body:
       A clear and concise description of what the bug is.
 
       ```python
-      # Sample code to reproduce the problem
+      # Sample code to reproduce the problem. If applicable, also include your model export command.
       ```
 
       ```

.github/scripts/extract_benchmark_results.py

@@ -229,21 +229,22 @@ def extract_ios_metric(
 
     elif method == "forward":
         if metric_name == "Clock Monotonic Time, s":
-            benchmark_result["metric"] = (
-                "generate_time(ms)"
-                if "llama" in test_name
-                else "avg_inference_latency(ms)"
-            )
+            benchmark_result["metric"] = "avg_inference_latency(ms)"
             benchmark_result["actualValue"] = metric_value * 1000
 
         elif metric_name == "Memory Peak Physical, kB":
             # NB: Showing the value in mB is friendlier IMO
             benchmark_result["metric"] = "peak_inference_mem_usage(mb)"
             benchmark_result["actualValue"] = metric_value / 1024
 
-    elif method == "generate" and metric_name == "Tokens Per Second, t/s":
-        benchmark_result["metric"] = "token_per_sec"
-        benchmark_result["actualValue"] = metric_value
+    elif method == "generate":
+        if metric_name == "Clock Monotonic Time, s":
+            benchmark_result["metric"] = "generate_time(ms)"
+            benchmark_result["actualValue"] = metric_value * 1000
+
+        elif metric_name == "Tokens Per Second, t/s":
+            benchmark_result["metric"] = "token_per_sec"
+            benchmark_result["actualValue"] = metric_value
 
     return benchmark_result
 

.github/workflows/_unittest.yml

@@ -7,6 +7,14 @@ on:
         required: true
         type: string
         description: Name of the docker image to use.
+      build-mode:
+        required: true
+        type: string
+        description: Build mode to use, Debug or Release.
+      build-tool:
+        required: true
+        type: string
+        description: Build tool to use, cmake or buck2.
       python-version:
         required: false
         type: string
@@ -26,28 +34,7 @@ jobs:
       timeout: 90
       script: |
         set -eux
-
-        # The generic Linux job chooses to use base env, not the one setup by the image
-        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
-        conda activate "${CONDA_ENV}"
-
-        # Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
-        source .ci/scripts/setup-vulkan-linux-deps.sh
-
-        # Setup MacOS dependencies as there is no Docker support on MacOS atm
-        PYTHON_EXECUTABLE=python \
-        EXECUTORCH_BUILD_PYBIND=ON \
-        CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
-        .ci/scripts/setup-linux.sh cmake
-
-        # Install llama3_2_vision dependencies.
-        PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
-
-        # Run pytest with coverage
-        pytest -n auto --cov=./ --cov-report=xml
-        # Run gtest
-        LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \
-        test/run_oss_cpp_tests.sh
+        .ci/scripts/unittest-linux.sh "${{ inputs.build-tool }}" "${{ inputs.build-mode }}"
 
   macos:
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
@@ -58,27 +45,4 @@ jobs:
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
         set -eux
-
-        bash .ci/scripts/setup-conda.sh
-
-        # Create temp directory for sccache shims
-        export TMP_DIR=$(mktemp -d)
-        export PATH="${TMP_DIR}:$PATH"
-        trap 'rm -rfv ${TMP_DIR}' EXIT
-
-        # Setup MacOS dependencies as there is no Docker support on MacOS atm
-        PYTHON_EXECUTABLE=python \
-        EXECUTORCH_BUILD_PYBIND=ON \
-        CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
-        ${CONDA_RUN} --no-capture-output \
-        .ci/scripts/setup-macos.sh cmake
-
-        # Install llama3_2_vision dependencies.
-        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
-        ./examples/models/llama3_2_vision/install_requirements.sh
-
-        # Run pytest with coverage
-        ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml
-        # Run gtest
-        LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \
-        ${CONDA_RUN} test/run_oss_cpp_tests.sh
+        .ci/scripts/unittest-macos.sh "${{ inputs.build-tool }}" "${{ inputs.build-mode }}"

.github/workflows/pull.yml

@@ -367,6 +367,18 @@ jobs:
       id-token: write
       contents: read
     with:
+      build-mode: Debug
+      build-tool: cmake
+      docker-image: executorch-ubuntu-22.04-clang12
+
+  unittest-buck:
+    uses: ./.github/workflows/_unittest.yml
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      build-mode: Debug
+      build-tool: buck2
       docker-image: executorch-ubuntu-22.04-clang12
 
   unittest-arm:

.github/workflows/trunk.yml

@@ -159,7 +159,7 @@ jobs:
         sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
 
         # Test ethos-u delegate examples with run.sh
-        backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp
+        backends/arm/test/test_arm_baremetal.sh test_full_ethosu_fvp
 
 
   test-arm-reference-delegation:
@@ -489,3 +489,13 @@ jobs:
         PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
         # Test llama2
         PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -mode "${MODE}" -dtype "${DTYPE}" -pt2e_quantize "${PT2E_QUANTIZE}"
+
+  unittest-release:
+    uses: ./.github/workflows/_unittest.yml
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      build-mode: Release
+      build-tool: cmake
+      docker-image: executorch-ubuntu-22.04-clang12

CMakeLists.txt

@@ -258,6 +258,11 @@ if(EXECUTORCH_BUILD_EXTENSION_TRAINING)
   set(EXECUTORCH_BUILD_EXTENSION_MODULE ON)
 endif()
 
+if(EXECUTORCH_BUILD_EXTENSION_MODULE)
+  set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON)
+  set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON)
+endif()
+
 if(EXECUTORCH_BUILD_KERNELS_CUSTOM_AOT)
   set(EXECUTORCH_BUILD_EXTENSION_TENSOR ON)
   set(EXECUTORCH_BUILD_KERNELS_CUSTOM ON)