Support sparse checkout (#5015)

* Handle sparse checkout YAML properties

* Fail task in case of issues that a user needs to address e.g. pattern format

src/Agent.Plugins/GitCliManager.cs

@@ -320,6 +320,40 @@ public async Task<int> GitLFSFetch(AgentTaskPluginExecutionContext context, stri
             return fetchExitCode;
         }
 
+        // git sparse-checkout
+        public async Task<int> GitSparseCheckout(AgentTaskPluginExecutionContext context, string repositoryPath, string directories, string patterns, CancellationToken cancellationToken)
+        {
+            context.Debug($"Sparse checkout");
+
+            bool useConeMode = !string.IsNullOrWhiteSpace(directories);
+            string options = useConeMode ? "--cone" : "--no-cone";
+
+            context.PublishTelemetry(area: "AzurePipelinesAgent", feature: "GitSparseCheckout", properties: new Dictionary<string, string>
+            {
+                { "Mode", useConeMode ? "cone" : "non-cone" },
+                { "Patterns", useConeMode ? directories : patterns }
+            });
+
+            int exitCode_sparseCheckoutInit = await ExecuteGitCommandAsync(context, repositoryPath, "sparse-checkout init", options, cancellationToken);
+
+            if (exitCode_sparseCheckoutInit != 0)
+            {
+                return exitCode_sparseCheckoutInit;
+            }
+            else
+            {
+                return await ExecuteGitCommandAsync(context, repositoryPath, "sparse-checkout set", useConeMode ? directories : patterns, cancellationToken);
+            }
+        }
+
+        // git sparse-checkout disable
+        public async Task<int> GitSparseCheckoutDisable(AgentTaskPluginExecutionContext context, string repositoryPath, CancellationToken cancellationToken)
+        {
+            context.Debug($"Sparse checkout disable");
+
+            return await ExecuteGitCommandAsync(context, repositoryPath, "sparse-checkout disable", string.Empty, cancellationToken);
+        }
+
         // git checkout -f --progress <commitId/branch>
         public async Task<int> GitCheckout(AgentTaskPluginExecutionContext context, string repositoryPath, string committishOrBranchSpec, string additionalCommandLine, CancellationToken cancellationToken)
         {

src/Agent.Plugins/GitSourceProvider.cs

@@ -342,6 +342,10 @@ public async Task GetSourceAsync(
 
             string fetchFilter = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.FetchFilter);
 
+            string sparseCheckoutDirectories = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.SparseCheckoutDirectories);
+            string sparseCheckoutPatterns = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.SparseCheckoutPatterns);
+            bool enableSparseCheckout = !string.IsNullOrWhiteSpace(sparseCheckoutDirectories) || !string.IsNullOrWhiteSpace(sparseCheckoutPatterns);
+
             executionContext.Debug($"repository url={repositoryUrl}");
             executionContext.Debug($"targetPath={targetPath}");
             executionContext.Debug($"sourceBranch={sourceBranch}");
@@ -355,6 +359,7 @@ public async Task GetSourceAsync(
             executionContext.Debug($"fetchTags={fetchTags}");
             executionContext.Debug($"gitLfsSupport={gitLfsSupport}");
             executionContext.Debug($"acceptUntrustedCerts={acceptUntrustedCerts}");
+            executionContext.Debug($"sparseCheckout={enableSparseCheckout}");
 
             bool schannelSslBackend = StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("agent.gituseschannel")?.Value);
             executionContext.Debug($"schannelSslBackend={schannelSslBackend}");
@@ -969,6 +974,30 @@ public async Task GetSourceAsync(
                 }
             }
 
+            if (AgentKnobs.UseSparseCheckoutInCheckoutTask.GetValue(executionContext).AsBoolean())
+            {
+                if (enableSparseCheckout)
+                {
+                    // Set up sparse checkout
+                    int exitCode_sparseCheckout = await gitCommandManager.GitSparseCheckout(executionContext, targetPath, sparseCheckoutDirectories, sparseCheckoutPatterns, cancellationToken);
+
+                    if (exitCode_sparseCheckout != 0)
+                    {
+                        throw new InvalidOperationException($"Git sparse checkout failed with exit code: {exitCode_sparseCheckout}");
+                    }
+                }
+                else
+                {
+                    // Disable sparse checkout in case it was enabled in a previous checkout
+                    int exitCode_sparseCheckoutDisable = await gitCommandManager.GitSparseCheckoutDisable(executionContext, targetPath, cancellationToken);
+
+                    if (exitCode_sparseCheckoutDisable != 0)
+                    {
+                        throw new InvalidOperationException($"Git sparse checkout disable failed with exit code: {exitCode_sparseCheckoutDisable}");
+                    }
+                }
+            }
+
             // Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw)
             int exitCode_checkout = await gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, string.Join(" ", additionalCheckoutArgs), cancellationToken);
             if (exitCode_checkout != 0)

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -773,5 +773,11 @@ public class AgentKnobs
             new EnvironmentKnobSource("AGENT_INSTALL_LEGACY_TF_EXE"),
             new PipelineFeatureSource("InstallLegacyTfExe"),
             new BuiltInDefaultKnobSource("false"));
+
+        public static readonly Knob UseSparseCheckoutInCheckoutTask = new Knob(
+            nameof(UseSparseCheckoutInCheckoutTask),
+            "If true, agent will use sparse checkout in checkout task.",
+            new RuntimeKnobSource("AGENT_USE_SPARSE_CHECKOUT_IN_CHECKOUT_TASK"),
+            new BuiltInDefaultKnobSource("false"));
     }
 }

src/Common.props

@@ -11,7 +11,7 @@
     <OSPlatform>OS_UNKNOWN</OSPlatform>
     <OSArchitecture>ARCH_UNKNOWN</OSArchitecture>
     <DebugConstant></DebugConstant>
-    <VssApiVersion>0.5.246-private</VssApiVersion>
+    <VssApiVersion>0.5.247-private</VssApiVersion>
     <CodeAnalysis>$(CodeAnalysis)</CodeAnalysis>
     <InvariantGlobalization>false</InvariantGlobalization>
     <EnforceCodeStyleInBuild>false</EnforceCodeStyleInBuild>