Skip to content

Commit

Permalink
Merged PR 41074: [internal/release/6.0.4xx] Merge from public
Browse files Browse the repository at this point in the history
Merge from public release/6.0.4xx to internal/release/6.0.4xx and resolve conflicts if necessary

----
#### AI description  (iteration 1)
#### PR Classification
Code enhancement and internal feature addition.

#### PR Summary
This pull request introduces new YAML templates for generating SAS tokens and federated access tokens, along with updates to existing build and release workflows to support internal sources and authentication.
- Added `get-delegation-sas.yml` and `get-federated-access-token.yml` templates for generating SAS tokens and access tokens.
- Updated `source-index-stage1.yml` and `source-build.yml` to include steps for Azure authentication and internal source enabling.
- Modified `azure-pipelines.yml` and `azure-pipelines-pr.yml` to integrate new internal runtime enabling steps.
- Updated `NuGet.config` to include new internal package sources.
- Bumped `Microsoft.DotNet.Arcade.Sdk` version in `Version.Details.xml`.
  • Loading branch information
vseanreesermsft committed Jul 15, 2024
2 parents 89f5155 + 0d275a4 commit 3d9c9f0
Show file tree
Hide file tree
Showing 21 changed files with 334 additions and 33 deletions.
13 changes: 13 additions & 0 deletions .github/workflows/inter-branch-merge-flow.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
name: Inter-branch merge workflow
on:
push:
branches:
- release/**

permissions:
contents: write
pull-requests: write

jobs:
Merge:
uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main
8 changes: 8 additions & 0 deletions NuGet.config
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@
<!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
<!-- Begin: Package sources from dotnet-runtime -->
<add key="darc-int-dotnet-runtime-e77011b" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-runtime-e77011b3/nuget/v3/index.json" />
<add key="darc-int-dotnet-runtime-e77011b-5" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-runtime-e77011b3-5/nuget/v3/index.json" />
<add key="darc-int-dotnet-runtime-e77011b-3" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-runtime-e77011b3-3/nuget/v3/index.json" />
<add key="darc-int-dotnet-runtime-e77011b-2" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-runtime-e77011b3-2/nuget/v3/index.json" />
<add key="darc-int-dotnet-runtime-e77011b-1" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-runtime-e77011b3-1/nuget/v3/index.json" />
<!-- End: Package sources from dotnet-runtime -->
<!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
<add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
Expand All @@ -19,6 +23,10 @@
<disabledPackageSources>
<!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
<!-- Begin: Package sources from dotnet-runtime -->
<add key="darc-int-dotnet-runtime-e77011b-1" value="true" />
<add key="darc-int-dotnet-runtime-e77011b-2" value="true" />
<add key="darc-int-dotnet-runtime-e77011b-3" value="true" />
<add key="darc-int-dotnet-runtime-e77011b-5" value="true" />
<add key="darc-int-dotnet-runtime-e77011b" value="true" />
<!-- End: Package sources from dotnet-runtime -->
<!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
Expand Down
14 changes: 12 additions & 2 deletions azure-pipelines-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,8 @@ variables:

- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: Templating-SDLValidation-Params

- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- group: DotNetBuilds storage account read tokens
- name: _InternalRuntimeDownloadArgs
value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal
/p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
Expand Down Expand Up @@ -56,6 +55,8 @@ stages:
enablePublishBuildAssets: true
enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }}
enableSourceBuild: true
sourceBuildParameters:
enableInternalSources: true
enableTelemetry: true
helixRepo: dotnet/templating
jobs:
Expand Down Expand Up @@ -111,6 +112,9 @@ stages:
arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)

- template: /eng/common/templates/steps/enable-internal-runtimes.yml

# Use utility script to run script command dependent on agent OS.
- script: eng/common/cibuild.cmd
-configuration $(_BuildConfig)
Expand Down Expand Up @@ -141,6 +145,9 @@ stages:
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)

- template: /eng/common/templates/steps/enable-internal-runtimes.yml

- script: eng/common/cibuild.sh
--configuration $(_BuildConfig)
--prepareMachine
Expand Down Expand Up @@ -174,6 +181,9 @@ stages:
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)

- template: /eng/common/templates/steps/enable-internal-runtimes.yml

- script: eng/common/cibuild.sh
--configuration $(_BuildConfig)
--prepareMachine
Expand Down
20 changes: 16 additions & 4 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ variables:
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: Templating-SDLValidation-Params
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- group: DotNetBuilds storage account read tokens
- name: _InternalRuntimeDownloadArgs
value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal
/p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
Expand Down Expand Up @@ -86,6 +85,8 @@ extends:
enablePublishBuildAssets: true
enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }}
enableSourceBuild: true
sourceBuildParameters:
enableInternalSources: true
enableTelemetry: true
helixRepo: dotnet/templating
jobs:
Expand All @@ -101,7 +102,7 @@ extends:
- _SignType: test
- ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
- _SignType: real
- _InternalBuildArgs: ''
- _InternalBuildArgs: ''
# Only enable publishing in non-public, non PR scenarios.
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
# DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat
Expand All @@ -122,6 +123,9 @@ extends:
arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)

- template: /eng/common/templates-official/steps/enable-internal-runtimes.yml

# Use utility script to run script command dependent on agent OS.
- script: eng/common/cibuild.cmd
-configuration $(_BuildConfig)
Expand All @@ -135,7 +139,9 @@ extends:
- job: OSX_${{ config.buildConfig }}
displayName: OSX ${{ config.buildConfig }}
pool:
name: 'Azure Pipelines'
vmImage: 'macOS-latest'
os: macOS
variables:
- _BuildConfig: ${{ config.buildConfig }}
- _SignType: none
Expand All @@ -148,14 +154,17 @@ extends:
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)

- template: /eng/common/templates-official/steps/enable-internal-runtimes.yml

- script: eng/common/cibuild.sh
--configuration $(_BuildConfig)
--prepareMachine
--integrationTest
$(_InternalRuntimeDownloadArgs)
name: Build
displayName: Build

- ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
- ${{ each config in parameters.buildConfigurations }}:
- job: Linux_${{ config.buildConfig }}
Expand All @@ -181,6 +190,9 @@ extends:
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)

- template: /eng/common/templates-official/steps/enable-internal-runtimes.yml

- script: eng/common/cibuild.sh
--configuration $(_BuildConfig)
--prepareMachine
Expand All @@ -189,7 +201,7 @@ extends:
name: Build
displayName: Build
condition: succeeded()

- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- template: eng/common/templates-official/post-build/post-build.yml@self
parameters:
Expand Down
4 changes: 2 additions & 2 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,9 @@
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.24270.1">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.24360.7">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2eab07c3d7b78219d10099b19fafeef2ecae1779</Sha>
<Sha>fbc993a9e8fb4926ce04c95ba2e48852c9d9df65</Sha>
<SourceBuild RepoName="arcade" ManagedOnly="true" />
</Dependency>
</ToolsetDependencies>
Expand Down
2 changes: 1 addition & 1 deletion eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<PropertyGroup>
<UsingToolXliff>true</UsingToolXliff>
<UsingToolNetFrameworkReferenceAssemblies>true</UsingToolNetFrameworkReferenceAssemblies>
<VersionPrefix>6.0.424</VersionPrefix>
<VersionPrefix>6.0.425</VersionPrefix>
<PreReleaseVersionLabel>rtm</PreReleaseVersionLabel>
<!--
When StabilizePackageVersion is set to 'true', this branch will produce stable outputs for 'Shipping' packages
Expand Down
8 changes: 8 additions & 0 deletions eng/common/templates-official/job/source-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ parameters:
# container and pool.
platform: {}

# If set to true and running on a non-public project,
# Internal blob storage locations will be enabled.
# This is not enabled by default because many repositories do not need internal sources
# and do not need to have the required service connections approved in the pipeline.
enableInternalSources: false

jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
Expand Down Expand Up @@ -59,6 +65,8 @@ jobs:
clean: all

steps:
- ${{ if eq(parameters.enableInternalSources, true) }}:
- template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
- template: /eng/common/templates-official/steps/source-build.yml
parameters:
platform: ${{ parameters.platform }}
35 changes: 25 additions & 10 deletions eng/common/templates-official/job/source-index-stage1.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20240320.1
sourceIndexUploadPackageVersion: 2.0.0-20240502.12
sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
Expand All @@ -17,14 +18,14 @@ jobs:
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
- name: SourceIndexPackageVersion
value: ${{ parameters.sourceIndexPackageVersion }}
- name: SourceIndexUploadPackageVersion
value: ${{ parameters.sourceIndexUploadPackageVersion }}
- name: SourceIndexProcessBinlogPackageVersion
value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
- name: SourceIndexPackageSource
value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables

pool: ${{ parameters.pool }}
steps:
Expand All @@ -40,8 +41,8 @@ jobs:
workingDirectory: $(Agent.TempDirectory)

- script: |
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
displayName: Download Tools
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)
Expand All @@ -53,7 +54,21 @@ jobs:
displayName: Process Binlog into indexable sln

- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
- task: AzureCLI@2
displayName: Get stage 1 auth token
inputs:
azureSubscription: 'SourceDotNet Stage1 Publish'
addSpnToEnvironment: true
scriptType: 'ps'
scriptLocation: 'inlineScript'
inlineScript: |
echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
- script: |
az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
displayName: "Login to Azure"
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
displayName: Upload stage1 artifacts to source index
env:
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
8 changes: 8 additions & 0 deletions eng/common/templates-official/jobs/source-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ parameters:
# one job runs on 'defaultManagedPlatform'.
platforms: []

# If set to true and running on a non-public project,
# Internal nuget and blob storage locations will be enabled.
# This is not enabled by default because many repositories do not need internal sources
# and do not need to have the required service connections approved in the pipeline.
enableInternalSources: false

jobs:

- ${{ if ne(parameters.allCompletedJobId, '') }}:
Expand All @@ -38,9 +44,11 @@ jobs:
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
enableInternalSources: ${{ parameters.enableInternalSources }}

- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates-official/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
enableInternalSources: ${{ parameters.enableInternalSources }}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ variables:
- group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage
- group: DotNet-MSRC-Storage
- group: Publish-Build-Assets

# Whether the build is internal or not
Expand Down
28 changes: 28 additions & 0 deletions eng/common/templates-official/steps/enable-internal-runtimes.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
# variable with the base64-encoded SAS token, by default

parameters:
- name: federatedServiceConnection
type: string
default: 'dotnetbuilds-internal-read'
- name: outputVariableName
type: string
default: 'dotnetbuilds-internal-container-read-token-base64'
- name: expiryInHours
type: number
default: 1
- name: base64Encode
type: boolean
default: true

steps:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- template: /eng/common/templates-official/steps/get-delegation-sas.yml
parameters:
federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
outputVariableName: ${{ parameters.outputVariableName }}
expiryInHours: ${{ parameters.expiryInHours }}
base64Encode: ${{ parameters.base64Encode }}
storageAccount: dotnetbuilds
container: internal
permissions: rl
43 changes: 43 additions & 0 deletions eng/common/templates-official/steps/get-delegation-sas.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
parameters:
- name: federatedServiceConnection
type: string
- name: outputVariableName
type: string
- name: expiryInHours
type: number
default: 1
- name: base64Encode
type: boolean
default: false
- name: storageAccount
type: string
- name: container
type: string
- name: permissions
type: string
default: 'rl'

steps:
- task: AzureCLI@2
displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
inputs:
azureSubscription: ${{ parameters.federatedServiceConnection }}
scriptType: 'pscore'
scriptLocation: 'inlineScript'
inlineScript: |
# Calculate the expiration of the SAS token and convert to UTC
$expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
$sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to generate SAS token."
exit 1
}
if ('${{ parameters.base64Encode }}' -eq 'true') {
$sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
}
Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"
Loading

0 comments on commit 3d9c9f0

Please sign in to comment.