Skip to content

Commit

Permalink
Merge pull request #105 from EnzymeAD/ap/comptime_nn
Browse files Browse the repository at this point in the history
perf: test compile times for neural networks -- VGG16
  • Loading branch information
avik-pal authored Sep 16, 2024
2 parents 46b946a + de6264e commit fa10aa0
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 64 deletions.
130 changes: 66 additions & 64 deletions .github/workflows/benchmark_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,70 +7,72 @@ permissions:
pull-requests: write

jobs:
generate_plots:
runs-on: ubuntu-latest
generate_plots:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
with:
version: "1"
- uses: julia-actions/cache@v2
- name: Extract Package Name from Project.toml
id: extract-package-name
run: |
PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
echo "::set-output name=package_name::$PACKAGE_NAME"
- name: Build AirspeedVelocity
env:
JULIA_NUM_THREADS: 2
run: |
# Lightweight build step, as sometimes the runner runs out of memory:
julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add("AirspeedVelocity")'
julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
- name: Add ~/.julia/bin to PATH
run: |
echo "$HOME/.julia/bin" >> $GITHUB_PATH
- name: Run benchmarks
run: |
echo $PATH
ls -l ~/.julia/bin
mkdir results
benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune --add Enzyme
- name: Create plots from benchmarks
run: |
mkdir -p plots
benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
- name: Upload plot as artifact
uses: actions/upload-artifact@v4
with:
name: plots
path: plots
- name: Create markdown table from benchmarks
run: |
benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
echo '### Benchmark Results' > body.md
echo '' >> body.md
echo '' >> body.md
cat table.md >> body.md
echo '' >> body.md
echo '' >> body.md
echo '### Benchmark Plots' >> body.md
echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
steps:
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
with:
version: "1"
- uses: julia-actions/cache@v2
- name: Extract Package Name from Project.toml
id: extract-package-name
run: |
PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
echo "::set-output name=package_name::$PACKAGE_NAME"
- name: Build AirspeedVelocity
env:
JULIA_NUM_THREADS: 2
run: |
# Lightweight build step, as sometimes the runner runs out of memory:
julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add("AirspeedVelocity")'
julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
- name: Add ~/.julia/bin to PATH
run: |
echo "$HOME/.julia/bin" >> $GITHUB_PATH
- name: Run benchmarks
run: |
echo $PATH
ls -l ~/.julia/bin
mkdir results
benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ -s="benchmark/benchmarks.jl" --tune --add="Enzyme,Lux,Boltz,Random"
env:
JULIA_PKG_SERVER: ""
- name: Create plots from benchmarks
run: |
mkdir -p plots
benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
- name: Upload plot as artifact
uses: actions/upload-artifact@v4
with:
name: plots
path: plots
- name: Create markdown table from benchmarks
run: |
benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
echo '### Benchmark Results' > body.md
echo '' >> body.md
echo '' >> body.md
cat table.md >> body.md
echo '' >> body.md
echo '' >> body.md
echo '### Benchmark Plots' >> body.md
echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
- name: Find Comment
uses: peter-evans/find-comment@v3
id: fcbenchmark
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: Benchmark Results
- name: Find Comment
uses: peter-evans/find-comment@v3
id: fcbenchmark
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: Benchmark Results

- name: Comment on PR
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-path: body.md
edit-mode: replace
- name: Comment on PR
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-path: body.md
edit-mode: replace
16 changes: 16 additions & 0 deletions benchmark/benchmarks.jl
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
using BenchmarkTools
using Reactant
using Enzyme
using Boltz, Lux, Random

const SUITE = BenchmarkGroup()

SUITE["comptime"] = BenchmarkGroup()

SUITE["comptime"]["basics"] = BenchmarkGroup()
SUITE["comptime"]["basics"]["2D sum"] = @benchmarkable Reactant.compile(sum, (a,)) setup = (
a = Reactant.ConcreteRArray(ones(2, 10))
Expand All @@ -21,6 +23,20 @@ SUITE["comptime"]["basics"]["Basic cos"] = @benchmarkable Reactant.compile(cos,
a = Reactant.ConcreteRArray(ones(2, 10))
)

SUITE["comptime"]["lux neural networks"] = BenchmarkGroup()

for depth in [11, 13, 16, 19]
SUITE["comptime"]["lux neural networks"]["vgg$depth"] = @benchmarkable begin
Reactant.compile(vgg, (x, ps_concrete, st_concrete))
end setup = begin
vgg = Vision.VGG($depth; pretrained=false, batchnorm=false)
ps, st = Lux.setup(Random.default_rng(), vgg)
ps_concrete = ps |> Reactant.to_rarray
st_concrete = st |> Lux.testmode |> Reactant.to_rarray
x = rand(Float32, 224, 224, 3, 16) |> Reactant.to_rarray
end
end

function sumcos(x)
return sum(cos.(x))
end
Expand Down

0 comments on commit fa10aa0

Please sign in to comment.