azure-pipelines.yml

resources:
  containers:
    - container: ubuntu_x64_build_container
      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20210311173856-047508b

# CI Trigger on main branch
trigger:
  branches:
    include:
    - main
    - release/3.1.4xx
  paths:
    include:
    - eng/Version.Details.xml

# Trigger builds for PR's targeting main
pr:
  branches:
    include:
    - main
  paths:
    exclude: # don't trigger the CI if only a doc file was changed
    - docs/*
    - eng/common/README.md
    - src/benchmarks/micro/Serializers/README.md
    - src/benchmarks/micro/README.md
    - src/benchmarks/real-world/JitBench/README.md
    - src/benchmarks/real-world/Microsoft.ML.Benchmarks/Input/README.md
    - src/tools/ResultsComparer/README.md
    - README.md

schedules:
- cron: "0 */12 * * *"
  displayName: Every 12 hours build
  branches:
    include:
    - main
    - release/3.1.4xx
  always: true

jobs:

######################################################
# Public correctness jobs
######################################################

- ${{ if eq(variables['System.TeamProject'], 'public') }}:
  # Windows x64 scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x64
      pool: Hosted VS2017
      kind: scenarios
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      projectFile: scenarios.proj
      channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks
        - main

  # Ubuntu 1804 x64 scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: scenarios
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Open
      queue: Ubuntu.1804.Amd64.Open
      container: ubuntu_x64_build_container
      projectFile: scenarios.proj
      channels:
        - main
  
  # Windows x64 Blazor scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x64
      pool: Hosted VS2017
      kind: blazor_scenarios
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      projectFile: blazor_scenarios.proj
      channels:
        - main
        - release/5.0.1xx

  # Windows x64 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x64
      pool: Hosted VS2017
      kind: sdk_scenarios
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      projectFile: sdk_scenarios.proj
      channels:
        - main
  
  # Ubuntu 1804 x64 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: sdk_scenarios
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Open
      queue: Ubuntu.1804.Amd64.Open
      container: ubuntu_x64_build_container
      projectFile: sdk_scenarios.proj
      channels:
        - main

  # Windows x86 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x86
      pool: Hosted VS2017
      kind: sdk_scenarios
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      projectFile: sdk_scenarios.proj
      channels:
        - main
  
  # Windows x64 micro benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: RS5
      kind: micro
      architecture: x64
      pool: Hosted VS2017
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      csproj: src\benchmarks\micro\MicroBenchmarks.csproj
      runCategories: 'runtime libraries' 
      channels:
        - main
        - release/5.0.1xx
        #- release/3.1.3xx
        #- 2.1

  # Windows x64 net461 micro benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: RS4
      kind: micro_net461
      architecture: x64
      pool: Hosted VS2017
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS4.DevEx.15.8.Open
      csproj: src\benchmarks\micro\MicroBenchmarks.csproj
      runCategories: 'runtime libraries'
      channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks
        - LTS # use LTS channel for net461 framework

  # Windows x86 micro benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: RS5
      kind: micro
      architecture: x86
      pool: Hosted VS2017
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      csproj: src\benchmarks\micro\MicroBenchmarks.csproj
      runCategories: 'runtime libraries'
      channels: # for public jobs we want to make sure that the PRs don't break x86
        - main
        - release/5.0.1xx

  # Windows x64 ML.NET benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: RS5
      kind: mlnet
      architecture: x64
      pool: Hosted VS2017
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
      runCategories: 'mldotnet'
      channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only
        - main
        - release/5.0.1xx
        #- release/3.1.3xx

  # Windows x64 Roslyn benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: RS5
      kind: roslyn
      architecture: x64
      pool: Hosted VS2017
      machinePool: Open
      queue: Windows.10.Amd64.ClientRS5.Open
      csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj
      runCategories: 'roslyn'
      channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only
        - main
        - release/5.0.1xx
        #- release/3.1.3xx
        
  # Ubuntu 1804 x64 micro benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: micro
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Open
      queue: Ubuntu.1804.Amd64.Open
      container: ubuntu_x64_build_container
      csproj: src/benchmarks/micro/MicroBenchmarks.csproj
      runCategories: 'runtime libraries'
      channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks
        - main
        - release/5.0.1xx
        #- release/3.1.3xx
        #- 2.1

  # Ubuntu 1804 x64 ML.NET benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: mlnet
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Open
      queue: Ubuntu.1804.Amd64.Open
      container: ubuntu_x64_build_container
      runCategories: 'mldotnet'
      csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
      channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only
        - main
        - release/5.0.1xx
        #- release/3.1.3xx

  # Ubuntu 1804 x64 Roslyn benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: roslyn
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Open
      queue: Ubuntu.1804.Amd64.Open
      container: ubuntu_x64_build_container
      runCategories: 'roslyn'
      csproj: src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj
      channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only
        - main
        - release/5.0.1xx
        #- release/3.1.3xx

###########################################
# Private Jobs
###########################################

- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')) }}:

  # Windows x64 scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: 19H1
      architecture: x64
      pool: Hosted VS2017
      kind: scenarios
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf
      projectFile: scenarios.proj
      channels: # for public jobs we want to make sure that the PRs don't break any of the supported channels
        - main

  # Ubuntu 1804 x64 scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: scenarios
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Tiger
      queue: Ubuntu.1804.Amd64.Tiger.Perf # using a dedicated private Helix queue (perftigers)
      container: ubuntu_x64_build_container
      projectFile: scenarios.proj
      channels:
        - main

  # Windows x64 micro benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: 19H1
      kind: micro
      architecture: x64
      pool: Hosted VS2017
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf # using a dedicated private Helix queue (perftigers)
      csproj: src\benchmarks\micro\MicroBenchmarks.csproj
      runCategories: 'runtime libraries'
      channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
        #- main
        - release/5.0.1xx
        #- release/3.1.2xx

  # # Windows AMD64 specific micro benchmarks
  # - template: /eng/performance/benchmark_jobs.yml
  #   parameters:
  #     osName: windows
  #     osVersion: 20H2
  #     kind: micro
  #     architecture: x64
  #     pool: Hosted VS2017
  #     machinePool: Owl
  #     queue: Windows.10.Amd64.20H2.Owl.Perf # using a dedicated private Helix queue (perfowls)
  #     csproj: src\benchmarks\micro\MicroBenchmarks.csproj
  #     runCategories: 'runtime libraries'
  #     channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
  #       #- main
  #       - release/5.0.1xx
  #       #- release/3.1.2xx
      
  # Windows x86 micro benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: 19H1
      kind: micro
      architecture: x86
      pool: Hosted VS2017
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf # using a dedicated private Helix queue (perftigers)
      csproj: src\benchmarks\micro\MicroBenchmarks.csproj
      runCategories: 'runtime libraries'
      channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
        #- main
        - release/5.0.1xx
        #- release/3.1.2xx

  # Windows x64 ML.NET benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: 19H1
      kind: mlnet
      architecture: x64
      pool: Hosted VS2017
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf # using a dedicated private Helix queue (perftigers)
      csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
      runCategories: 'mldotnet'
      channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
        #- main
        - release/5.0.1xx
        #- release/3.1.2xx

  # Windows x64 Roslyn benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: windows
      osVersion: 19H1
      kind: roslyn
      architecture: x64
      pool: Hosted VS2017
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf # using a dedicated private Helix queue (perfsnakes)
      csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj
      runCategories: 'roslyn'
      channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
        #- main
        - release/5.0.1xx
        #- release/3.1.2xx

  # Ubuntu 1804 x64 micro benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: micro
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Tiger
      queue: Ubuntu.1804.Amd64.Tiger.Perf # using a dedicated private Helix queue (perftigers)
      container: ubuntu_x64_build_container
      csproj: src/benchmarks/micro/MicroBenchmarks.csproj
      runCategories: 'runtime libraries'
      channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
        #- main
        - release/5.0.1xx
        #- release/3.1.2xx

  # # Ubuntu 1804 AMD64 specific micro benchmarks
  # - template: /eng/performance/benchmark_jobs.yml
  #   parameters:
  #     osName: ubuntu
  #     osVersion: 1804
  #     kind: micro
  #     architecture: x64
  #     pool: Hosted Ubuntu 1604
  #     machinePool: Owl
  #     queue: Ubuntu.1804.Amd64.Owl.Perf # using a dedicated private Helix queue (perfowls)
  #     container: ubuntu_x64_build_container
  #     csproj: src/benchmarks/micro/MicroBenchmarks.csproj
  #     runCategories: 'runtime libraries'
  #     channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
  #       #- main
  #       - release/5.0.1xx
  #       #- release/3.1.2xx

  # Ubuntu 1804 x64 ML.NET benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: mlnet
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Tiger
      queue: Ubuntu.1804.Amd64.Tiger.Perf # using a dedicated private Helix queue (perftigers)
      container: ubuntu_x64_build_container
      csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
      runCategories: 'mldotnet'
      channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
        #- main
        - release/5.0.1xx
        #- release/3.1.2xx
  
  # Ubuntu 1804 x64 Roslyn benchmarks
  - template: /eng/performance/benchmark_jobs.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      kind: roslyn
      architecture: x64
      pool: Hosted Ubuntu 1604
      machinePool: Tiger
      queue: Ubuntu.1804.Amd64.Tiger.Perf # using a dedicated private Helix queue (perftigers)
      container: ubuntu_x64_build_container
      csproj: src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj
      runCategories: 'roslyn'
      channels: # for private jobs we want to benchmark .NET Core 3.1 and 5.0 only
        #- main
        - release/5.0.1xx
        #- release/3.1.2xx

################################################
# Scheduled Private jobs
################################################

# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job
- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}:

  # Windows x64 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x64
      pool: Hosted VS2017
      kind: sdk_scenarios
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf
      projectFile: sdk_scenarios.proj
      channels:
        - main
        - 5.0
  
  # Windows x86 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x86
      pool: Hosted VS2017
      kind: sdk_scenarios
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf
      projectFile: sdk_scenarios.proj
      channels:
        - main

  # Ubuntu 1804 x64 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      architecture: x64
      pool: Hosted Ubuntu 1604
      kind: sdk_scenarios
      machinePool: Tiger
      queue: Ubuntu.1804.Amd64.Tiger.Perf
      container: ubuntu_x64_build_container
      projectFile: sdk_scenarios.proj
      channels:
        - main
        - 5.0

  # Windows x64 Blazor 3.2 scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x64
      pool: Hosted VS2017
      kind: blazor_scenarios
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf
      projectFile: blazor_scenarios.proj
      channels:
        - main
        #- release/5.0.1xx

################################################
# Manually Triggered Job
################################################


- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual')) }}:

  # Windows x64 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x64
      pool: Hosted VS2017
      kind: sdk_scenarios
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf
      projectFile: sdk_scenarios.proj
      channels:
        - $(channel) # for manual runs we can specify channel variable 

  # Windows x86 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: windows
      osVersion: RS5
      architecture: x86
      pool: Hosted VS2017
      kind: sdk_scenarios
      machinePool: Tiger
      queue: Windows.10.Amd64.19H1.Tiger.Perf
      projectFile: sdk_scenarios.proj
      channels:
        - $(channel) # for manual runs we can specify channel variable 

  # Ubuntu 1804 x64 SDK scenario benchmarks
  - template: /eng/performance/scenarios.yml
    parameters:
      osName: ubuntu
      osVersion: 1804
      architecture: x64
      pool: Hosted Ubuntu 1604
      kind: sdk_scenarios
      machinePool: Tiger
      queue: Ubuntu.1804.Amd64.Tiger.Perf
      container: ubuntu_x64_build_container
      projectFile: sdk_scenarios.proj
      channels:
        - $(channel)