From 2b55375727620771394ffec0372dfdc3e9e62fdf Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 6 Jan 2025 23:20:46 +0100 Subject: [PATCH 1/4] [Internal] Migrate workflows that need write access to use hosted runners (#1112) Fixes #1111. --- .github/workflows/external-message.yml | 9 +++++--- .github/workflows/integration-tests.yml | 30 +++++++++++++++++-------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml index 2d32dde9..80b15d65 100644 --- a/.github/workflows/external-message.yml +++ b/.github/workflows/external-message.yml @@ -13,7 +13,10 @@ on: jobs: comment-on-pr: - runs-on: ubuntu-latest + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + permissions: pull-requests: write @@ -44,13 +47,13 @@ jobs: gh pr comment ${{ github.event.pull_request.number }} --body \ " If integration tests don't run automatically, an authorized user can run them manually by following the instructions below: - + Trigger: [go/deco-tests-run/sdk-go](https://go/deco-tests-run/sdk-go) Inputs: * PR number: ${{github.event.pull_request.number}} * Commit SHA: \`${{ env.COMMIT_SHA }}\` - + Checks will be approved automatically on success. " diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 30919a1a..d0e8707c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -6,12 +6,16 @@ on: types: [opened, synchronize] merge_group: - + jobs: check-token: name: Check secrets access - runs-on: ubuntu-latest + + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + environment: "test-trigger-is" outputs: has_token: ${{ steps.set-token-status.outputs.has_token }} @@ -29,11 +33,15 @@ jobs: trigger-tests: name: Trigger Tests - runs-on: ubuntu-latest + + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + needs: check-token if: github.event_name == 'pull_request' && needs.check-token.outputs.has_token == 'true' environment: "test-trigger-is" - + steps: - uses: actions/checkout@v3 @@ -45,7 +53,7 @@ jobs: private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }} owner: ${{ secrets.ORG_NAME }} repositories: ${{secrets.REPO_NAME}} - + - name: Trigger Workflow in Another Repo env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} @@ -53,14 +61,18 @@ jobs: gh workflow run sdk-go-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \ --ref main \ -f pull_request_number=${{ github.event.pull_request.number }} \ - -f commit_sha=${{ github.event.pull_request.head.sha }} + -f commit_sha=${{ github.event.pull_request.head.sha }} + - # The hash for the merge queue may not be the same as the hash for the PR. # Auto approve the check for the merge queue to avoid running integration tests twice. auto-approve: if: github.event_name == 'merge_group' - runs-on: ubuntu-latest + + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + steps: - name: Mark Check env: @@ -71,4 +83,4 @@ jobs: -H "X-GitHub-Api-Version: 2022-11-28" \ /repos/${{ github.repository }}/statuses/${{ github.sha }} \ -f 'state=success' \ - -f 'context=Integration Tests Check' \ No newline at end of file + -f 'context=Integration Tests Check' From 83db3cbdab1ee3ccc4ec815d102d43e6c8e06bfe Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Tue, 7 Jan 2025 10:26:24 +0100 Subject: [PATCH 2/4] [Internal] Create custom codeql.yml (#1114) ## What changes are proposed in this pull request? With the recent security hardening, we can only use Databricks managed Github runners. This Repo currently uses the general `CodeQL` setup. Adding a custom codeql.yml to modify the runner group in the current setup. ## How is this tested? This workflow is a test. --- .github/workflows/codeql.yml | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..52c1cc12 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,42 @@ +name: "CodeQL Advanced" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '41 1 * * 0' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + permissions: + # required for all workflows + security-events: write + + strategy: + fail-fast: false + matrix: + include: + - language: go + build-mode: autobuild + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From adc94cabf7f3e21aa2b705f3fd799ffb5bf2f7f6 Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Tue, 7 Jan 2025 11:56:04 +0100 Subject: [PATCH 3/4] [Internal] Decouple serving and oauth2 package (#1110) ## What changes are proposed in this pull request? Package `oauth2` contains a `DataPlaneService` that is not intended for users to use directly. This service is not auto-generated and is only used in the `serving` package. To remove the dependency between the `serving` package and the `oauth2` package, this PR moves the `DataPlaneService` from the `oauth2` to the `serving` package. ## How is this tested? Existing unit tests --- .codegen/_openapi_sha | 2 +- .gitattributes | 2 + service/apps/model.go | 12 +++ service/catalog/model.go | 6 +- service/jobs/api.go | 26 +++++- service/jobs/interface.go | 14 +++- service/jobs/model.go | 83 ++++++++++++++++--- service/oauth2/model.go | 45 +++++----- service/pipelines/model.go | 39 +++++++++ service/pkg.go | 6 +- service/serving/api.go | 3 +- service/{oauth2 => serving}/ext_data_plane.go | 14 ++-- .../ext_data_plane_test.go | 26 +++--- service/serving/impl.go | 8 +- service/serving/model.go | 20 ++++- 15 files changed, 231 insertions(+), 75 deletions(-) rename service/{oauth2 => serving}/ext_data_plane.go (80%) rename service/{oauth2 => serving}/ext_data_plane_test.go (89%) diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 8622b29c..e43e9607 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d \ No newline at end of file +1668b0db17e23605f8c9d29fb3b674c01590732d \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index da818989..91ba6180 100644 --- a/.gitattributes +++ b/.gitattributes @@ -77,9 +77,11 @@ experimental/mocks/service/marketplace/mock_provider_provider_analytics_dashboar experimental/mocks/service/marketplace/mock_provider_providers_interface.go linguist-generated=true experimental/mocks/service/ml/mock_experiments_interface.go linguist-generated=true experimental/mocks/service/ml/mock_model_registry_interface.go linguist-generated=true +experimental/mocks/service/oauth2/mock_account_federation_policy_interface.go linguist-generated=true experimental/mocks/service/oauth2/mock_custom_app_integration_interface.go linguist-generated=true experimental/mocks/service/oauth2/mock_o_auth_published_apps_interface.go linguist-generated=true experimental/mocks/service/oauth2/mock_published_app_integration_interface.go linguist-generated=true +experimental/mocks/service/oauth2/mock_service_principal_federation_policy_interface.go linguist-generated=true experimental/mocks/service/oauth2/mock_service_principal_secrets_interface.go linguist-generated=true experimental/mocks/service/pipelines/mock_pipelines_interface.go linguist-generated=true experimental/mocks/service/provisioning/mock_credentials_interface.go linguist-generated=true diff --git a/service/apps/model.go b/service/apps/model.go index 7eaa5f90..0bced9bc 100755 --- a/service/apps/model.go +++ b/service/apps/model.go @@ -605,6 +605,18 @@ type CreateAppDeploymentRequest struct { // Create an app type CreateAppRequest struct { App *App `json:"app,omitempty"` + // If true, the app will not be started after creation. + NoCompute bool `json:"-" url:"no_compute,omitempty"` + + ForceSendFields []string `json:"-"` +} + +func (s *CreateAppRequest) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s CreateAppRequest) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) } // Delete an app diff --git a/service/catalog/model.go b/service/catalog/model.go index fce115ed..4493e9b3 100755 --- a/service/catalog/model.go +++ b/service/catalog/model.go @@ -4657,6 +4657,8 @@ type ProvisioningInfoState string const ProvisioningInfoStateActive ProvisioningInfoState = `ACTIVE` +const ProvisioningInfoStateDegraded ProvisioningInfoState = `DEGRADED` + const ProvisioningInfoStateDeleting ProvisioningInfoState = `DELETING` const ProvisioningInfoStateFailed ProvisioningInfoState = `FAILED` @@ -4673,11 +4675,11 @@ func (f *ProvisioningInfoState) String() string { // Set raw string value and validate it against allowed values func (f *ProvisioningInfoState) Set(v string) error { switch v { - case `ACTIVE`, `DELETING`, `FAILED`, `PROVISIONING`, `UPDATING`: + case `ACTIVE`, `DEGRADED`, `DELETING`, `FAILED`, `PROVISIONING`, `UPDATING`: *f = ProvisioningInfoState(v) return nil default: - return fmt.Errorf(`value "%s" is not one of "ACTIVE", "DELETING", "FAILED", "PROVISIONING", "UPDATING"`, v) + return fmt.Errorf(`value "%s" is not one of "ACTIVE", "DEGRADED", "DELETING", "FAILED", "PROVISIONING", "UPDATING"`, v) } } diff --git a/service/jobs/api.go b/service/jobs/api.go index b34dbfec..fb4f0ef5 100755 --- a/service/jobs/api.go +++ b/service/jobs/api.go @@ -81,11 +81,23 @@ type JobsInterface interface { // Get a single job. // // Retrieves the details for a single job. + // + // In Jobs API 2.2, requests for a single job support pagination of `tasks` and + // `job_clusters` when either exceeds 100 elements. Use the `next_page_token` + // field to check for more results and pass its value as the `page_token` in + // subsequent requests. Arrays with fewer than 100 elements in a page will be + // empty on later pages. Get(ctx context.Context, request GetJobRequest) (*Job, error) // Get a single job. // // Retrieves the details for a single job. + // + // In Jobs API 2.2, requests for a single job support pagination of `tasks` and + // `job_clusters` when either exceeds 100 elements. Use the `next_page_token` + // field to check for more results and pass its value as the `page_token` in + // subsequent requests. Arrays with fewer than 100 elements in a page will be + // empty on later pages. GetByJobId(ctx context.Context, jobId int64) (*Job, error) // Get job permission levels. @@ -112,7 +124,13 @@ type JobsInterface interface { // Get a single job run. // - // Retrieve the metadata of a run. + // Retrieves the metadata of a run. + // + // In Jobs API 2.2, requests for a single job run support pagination of `tasks` + // and `job_clusters` when either exceeds 100 elements. Use the + // `next_page_token` field to check for more results and pass its value as the + // `page_token` in subsequent requests. Arrays with fewer than 100 elements in a + // page will be empty on later pages. GetRun(ctx context.Context, request GetRunRequest) (*Run, error) // Get the output for a single run. @@ -430,6 +448,12 @@ func (a *JobsAPI) DeleteRunByRunId(ctx context.Context, runId int64) error { // Get a single job. // // Retrieves the details for a single job. +// +// In Jobs API 2.2, requests for a single job support pagination of `tasks` and +// `job_clusters` when either exceeds 100 elements. Use the `next_page_token` +// field to check for more results and pass its value as the `page_token` in +// subsequent requests. Arrays with fewer than 100 elements in a page will be +// empty on later pages. func (a *JobsAPI) GetByJobId(ctx context.Context, jobId int64) (*Job, error) { return a.jobsImpl.Get(ctx, GetJobRequest{ JobId: jobId, diff --git a/service/jobs/interface.go b/service/jobs/interface.go index b1ce89b9..be9b5518 100755 --- a/service/jobs/interface.go +++ b/service/jobs/interface.go @@ -61,6 +61,12 @@ type JobsService interface { // Get a single job. // // Retrieves the details for a single job. + // + // In Jobs API 2.2, requests for a single job support pagination of `tasks` + // and `job_clusters` when either exceeds 100 elements. Use the + // `next_page_token` field to check for more results and pass its value as + // the `page_token` in subsequent requests. Arrays with fewer than 100 + // elements in a page will be empty on later pages. Get(ctx context.Context, request GetJobRequest) (*Job, error) // Get job permission levels. @@ -76,7 +82,13 @@ type JobsService interface { // Get a single job run. // - // Retrieve the metadata of a run. + // Retrieves the metadata of a run. + // + // In Jobs API 2.2, requests for a single job run support pagination of + // `tasks` and `job_clusters` when either exceeds 100 elements. Use the + // `next_page_token` field to check for more results and pass its value as + // the `page_token` in subsequent requests. Arrays with fewer than 100 + // elements in a page will be empty on later pages. GetRun(ctx context.Context, request GetRunRequest) (*Run, error) // Get the output for a single run. diff --git a/service/jobs/model.go b/service/jobs/model.go index 641d98af..e23efe94 100755 --- a/service/jobs/model.go +++ b/service/jobs/model.go @@ -23,6 +23,11 @@ type BaseJob struct { // based on accessible budget policies of the run_as identity on job // creation or modification. EffectiveBudgetPolicyId string `json:"effective_budget_policy_id,omitempty"` + // Indicates if the job has more sub-resources (`tasks`, `job_clusters`) + // that are not shown. They can be accessed via :method:jobs/get endpoint. + // It is only relevant for API 2.2 :method:jobs/list requests with + // `expand_tasks=true`. + HasMore bool `json:"has_more,omitempty"` // The canonical identifier for this job. JobId int64 `json:"job_id,omitempty"` // Settings for this job and all of its runs. These settings can be updated @@ -89,9 +94,16 @@ type BaseRun struct { // Note: dbt and SQL File tasks support only version-controlled sources. If // dbt or SQL File tasks are used, `git_source` must be defined on the job. GitSource *GitSource `json:"git_source,omitempty"` + // Indicates if the run has more sub-resources (`tasks`, `job_clusters`) + // that are not shown. They can be accessed via :method:jobs/getrun + // endpoint. It is only relevant for API 2.2 :method:jobs/listruns requests + // with `expand_tasks=true`. + HasMore bool `json:"has_more,omitempty"` // A list of job cluster specifications that can be shared and reused by // tasks of this job. Libraries cannot be declared in a shared job cluster. - // You must declare dependent libraries in task settings. + // You must declare dependent libraries in task settings. If more than 100 + // job clusters are available, you can paginate through them using + // :method:jobs/getrun. JobClusters []JobCluster `json:"job_clusters,omitempty"` // The canonical identifier of the job that contains this run. JobId int64 `json:"job_id,omitempty"` @@ -153,7 +165,10 @@ type BaseRun struct { // The current status of the run Status *RunStatus `json:"status,omitempty"` // The list of tasks performed by the run. Each task has its own `run_id` - // which you can use to call `JobsGetOutput` to retrieve the run resutls. + // which you can use to call `JobsGetOutput` to retrieve the run resutls. If + // more than 100 tasks are available, you can paginate through them using + // :method:jobs/getrun. Use the `next_page_token` field at the object root + // to determine if more results are available. Tasks []RunTask `json:"tasks,omitempty"` // The type of trigger that fired this run. // @@ -543,7 +558,9 @@ type CreateJob struct { Health *JobsHealthRules `json:"health,omitempty"` // A list of job cluster specifications that can be shared and reused by // tasks of this job. Libraries cannot be declared in a shared job cluster. - // You must declare dependent libraries in task settings. + // You must declare dependent libraries in task settings. If more than 100 + // job clusters are available, you can paginate through them using + // :method:jobs/get. JobClusters []JobCluster `json:"job_clusters,omitempty"` // An optional maximum allowed number of concurrent runs of the job. Set // this value if you want to be able to execute multiple runs of the same @@ -583,7 +600,10 @@ type CreateJob struct { // limitations as cluster tags. A maximum of 25 tags can be added to the // job. Tags map[string]string `json:"tags,omitempty"` - // A list of task specifications to be executed by this job. + // A list of task specifications to be executed by this job. If more than + // 100 tasks are available, you can paginate through them using + // :method:jobs/get. Use the `next_page_token` field at the object root to + // determine if more results are available. Tasks []Task `json:"tasks,omitempty"` // An optional timeout applied to each run of this job. A value of `0` means // no timeout. @@ -961,6 +981,19 @@ type GetJobRequest struct { // The canonical identifier of the job to retrieve information about. This // field is required. JobId int64 `json:"-" url:"job_id"` + // Use `next_page_token` returned from the previous GetJob to request the + // next page of the job's sub-resources. + PageToken string `json:"-" url:"page_token,omitempty"` + + ForceSendFields []string `json:"-"` +} + +func (s *GetJobRequest) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s GetJobRequest) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) } // Get job policy compliance @@ -1005,8 +1038,8 @@ type GetRunRequest struct { IncludeHistory bool `json:"-" url:"include_history,omitempty"` // Whether to include resolved parameter values in the response. IncludeResolvedValues bool `json:"-" url:"include_resolved_values,omitempty"` - // To list the next page of job tasks, set this field to the value of the - // `next_page_token` returned in the GetJob response. + // Use `next_page_token` returned from the previous GetRun to request the + // next page of the run's sub-resources. PageToken string `json:"-" url:"page_token,omitempty"` // The canonical identifier of the run for which to retrieve the metadata. // This field is required. @@ -1139,8 +1172,15 @@ type Job struct { // based on accessible budget policies of the run_as identity on job // creation or modification. EffectiveBudgetPolicyId string `json:"effective_budget_policy_id,omitempty"` + // Indicates if the job has more sub-resources (`tasks`, `job_clusters`) + // that are not shown. They can be accessed via :method:jobs/get endpoint. + // It is only relevant for API 2.2 :method:jobs/list requests with + // `expand_tasks=true`. + HasMore bool `json:"has_more,omitempty"` // The canonical identifier for this job. JobId int64 `json:"job_id,omitempty"` + // A token that can be used to list the next page of sub-resources. + NextPageToken string `json:"next_page_token,omitempty"` // The email of an active workspace user or the application ID of a service // principal that the job runs as. This value can be changed by setting the // `run_as` field when creating or updating a job. @@ -1583,7 +1623,9 @@ type JobSettings struct { Health *JobsHealthRules `json:"health,omitempty"` // A list of job cluster specifications that can be shared and reused by // tasks of this job. Libraries cannot be declared in a shared job cluster. - // You must declare dependent libraries in task settings. + // You must declare dependent libraries in task settings. If more than 100 + // job clusters are available, you can paginate through them using + // :method:jobs/get. JobClusters []JobCluster `json:"job_clusters,omitempty"` // An optional maximum allowed number of concurrent runs of the job. Set // this value if you want to be able to execute multiple runs of the same @@ -1623,7 +1665,10 @@ type JobSettings struct { // limitations as cluster tags. A maximum of 25 tags can be added to the // job. Tags map[string]string `json:"tags,omitempty"` - // A list of task specifications to be executed by this job. + // A list of task specifications to be executed by this job. If more than + // 100 tasks are available, you can paginate through them using + // :method:jobs/get. Use the `next_page_token` field at the object root to + // determine if more results are available. Tasks []Task `json:"tasks,omitempty"` // An optional timeout applied to each run of this job. A value of `0` means // no timeout. @@ -1862,7 +1907,9 @@ func (s ListJobComplianceRequest) MarshalJSON() ([]byte, error) { // List jobs type ListJobsRequest struct { - // Whether to include task and cluster details in the response. + // Whether to include task and cluster details in the response. Note that in + // API 2.2, only the first 100 elements will be shown. Use :method:jobs/get + // to paginate through all tasks and clusters. ExpandTasks bool `json:"-" url:"expand_tasks,omitempty"` // The number of jobs to return. This value must be greater than 0 and less // or equal to 100. The default value is 20. @@ -1924,7 +1971,9 @@ type ListRunsRequest struct { // results; otherwise, lists both active and completed runs. This field // cannot be `true` when active_only is `true`. CompletedOnly bool `json:"-" url:"completed_only,omitempty"` - // Whether to include task and cluster details in the response. + // Whether to include task and cluster details in the response. Note that in + // API 2.2, only the first 100 elements will be shown. Use + // :method:jobs/getrun to paginate through all tasks and clusters. ExpandTasks bool `json:"-" url:"expand_tasks,omitempty"` // The job for which to list runs. If omitted, the Jobs service lists runs // from all jobs. @@ -2557,12 +2606,19 @@ type Run struct { // Note: dbt and SQL File tasks support only version-controlled sources. If // dbt or SQL File tasks are used, `git_source` must be defined on the job. GitSource *GitSource `json:"git_source,omitempty"` + // Indicates if the run has more sub-resources (`tasks`, `job_clusters`) + // that are not shown. They can be accessed via :method:jobs/getrun + // endpoint. It is only relevant for API 2.2 :method:jobs/listruns requests + // with `expand_tasks=true`. + HasMore bool `json:"has_more,omitempty"` // Only populated by for-each iterations. The parent for-each task is // located in tasks array. Iterations []RunTask `json:"iterations,omitempty"` // A list of job cluster specifications that can be shared and reused by // tasks of this job. Libraries cannot be declared in a shared job cluster. - // You must declare dependent libraries in task settings. + // You must declare dependent libraries in task settings. If more than 100 + // job clusters are available, you can paginate through them using + // :method:jobs/getrun. JobClusters []JobCluster `json:"job_clusters,omitempty"` // The canonical identifier of the job that contains this run. JobId int64 `json:"job_id,omitempty"` @@ -2626,7 +2682,10 @@ type Run struct { // The current status of the run Status *RunStatus `json:"status,omitempty"` // The list of tasks performed by the run. Each task has its own `run_id` - // which you can use to call `JobsGetOutput` to retrieve the run resutls. + // which you can use to call `JobsGetOutput` to retrieve the run resutls. If + // more than 100 tasks are available, you can paginate through them using + // :method:jobs/getrun. Use the `next_page_token` field at the object root + // to determine if more results are available. Tasks []RunTask `json:"tasks,omitempty"` // The type of trigger that fired this run. // diff --git a/service/oauth2/model.go b/service/oauth2/model.go index 5ce0f66d..08de08f9 100755 --- a/service/oauth2/model.go +++ b/service/oauth2/model.go @@ -9,8 +9,9 @@ import ( // Create account federation policy type CreateAccountFederationPolicyRequest struct { Policy *FederationPolicy `json:"policy,omitempty"` - // The identifier for the federation policy. If unspecified, the id will be - // assigned by Databricks. + // The identifier for the federation policy. The identifier must contain + // only lowercase alphanumeric characters, numbers, hyphens, and slashes. If + // unspecified, the id will be assigned by Databricks. PolicyId string `json:"-" url:"policy_id,omitempty"` ForceSendFields []string `json:"-"` @@ -105,8 +106,9 @@ func (s CreatePublishedAppIntegrationOutput) MarshalJSON() ([]byte, error) { // Create service principal federation policy type CreateServicePrincipalFederationPolicyRequest struct { Policy *FederationPolicy `json:"policy,omitempty"` - // The identifier for the federation policy. If unspecified, the id will be - // assigned by Databricks. + // The identifier for the federation policy. The identifier must contain + // only lowercase alphanumeric characters, numbers, hyphens, and slashes. If + // unspecified, the id will be assigned by Databricks. PolicyId string `json:"-" url:"policy_id,omitempty"` // The service principal id for the federation policy. ServicePrincipalId int64 `json:"-" url:"-"` @@ -153,25 +155,9 @@ func (s CreateServicePrincipalSecretResponse) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -type DataPlaneInfo struct { - // Authorization details as a string. - AuthorizationDetails string `json:"authorization_details,omitempty"` - // The URL of the endpoint for this operation in the dataplane. - EndpointUrl string `json:"endpoint_url,omitempty"` - - ForceSendFields []string `json:"-"` -} - -func (s *DataPlaneInfo) UnmarshalJSON(b []byte) error { - return marshal.Unmarshal(b, s) -} - -func (s DataPlaneInfo) MarshalJSON() ([]byte, error) { - return marshal.Marshal(s) -} - // Delete account federation policy type DeleteAccountFederationPolicyRequest struct { + // The identifier for the federation policy. PolicyId string `json:"-" url:"-"` } @@ -196,6 +182,7 @@ type DeleteResponse struct { // Delete service principal federation policy type DeleteServicePrincipalFederationPolicyRequest struct { + // The identifier for the federation policy. PolicyId string `json:"-" url:"-"` // The service principal id for the federation policy. ServicePrincipalId int64 `json:"-" url:"-"` @@ -214,9 +201,13 @@ type FederationPolicy struct { CreateTime string `json:"create_time,omitempty"` // Description of the federation policy. Description string `json:"description,omitempty"` - // Name of the federation policy. The name must contain only lowercase - // alphanumeric characters, numbers, and hyphens. It must be unique within - // the account. + // Resource name for the federation policy. Example values include + // `accounts//federationPolicies/my-federation-policy` for + // Account Federation Policies, and + // `accounts//servicePrincipals//federationPolicies/my-federation-policy` + // for Service Principal Federation Policies. Typically an output parameter, + // which does not need to be specified in create or update requests. If + // specified in a request, must match the value in the request URL. Name string `json:"name,omitempty"` // Specifies the policy to use for validating OIDC claims in your federated // tokens. @@ -239,6 +230,7 @@ func (s FederationPolicy) MarshalJSON() ([]byte, error) { // Get account federation policy type GetAccountFederationPolicyRequest struct { + // The identifier for the federation policy. PolicyId string `json:"-" url:"-"` } @@ -366,6 +358,7 @@ func (s GetPublishedAppsOutput) MarshalJSON() ([]byte, error) { // Get service principal federation policy type GetServicePrincipalFederationPolicyRequest struct { + // The identifier for the federation policy. PolicyId string `json:"-" url:"-"` // The service principal id for the federation policy. ServicePrincipalId int64 `json:"-" url:"-"` @@ -630,7 +623,7 @@ func (s TokenAccessPolicy) MarshalJSON() ([]byte, error) { // Update account federation policy type UpdateAccountFederationPolicyRequest struct { Policy *FederationPolicy `json:"policy,omitempty"` - + // The identifier for the federation policy. PolicyId string `json:"-" url:"-"` // Field mask is required to be passed into the PATCH request. Field mask // specifies which fields of the setting payload will be updated. The field @@ -663,7 +656,7 @@ type UpdatePublishedAppIntegrationOutput struct { // Update service principal federation policy type UpdateServicePrincipalFederationPolicyRequest struct { Policy *FederationPolicy `json:"policy,omitempty"` - + // The identifier for the federation policy. PolicyId string `json:"-" url:"-"` // The service principal id for the federation policy. ServicePrincipalId int64 `json:"-" url:"-"` diff --git a/service/pipelines/model.go b/service/pipelines/model.go index 3a67b316..04f11084 100755 --- a/service/pipelines/model.go +++ b/service/pipelines/model.go @@ -56,6 +56,13 @@ type CreatePipeline struct { Photon bool `json:"photon,omitempty"` // Restart window of this pipeline. RestartWindow *RestartWindow `json:"restart_window,omitempty"` + // Write-only setting, available only in Create/Update calls. Specifies the + // user or service principal that the pipeline runs as. If not specified, + // the pipeline runs as the user who created the pipeline. + // + // Only `user_name` or `service_principal_name` can be specified. If both + // are specified, an error is thrown. + RunAs *RunAs `json:"run_as,omitempty"` // The default schema (database) where tables are read from or published to. // The presence of this field implies that the pipeline is in direct // publishing mode. @@ -219,6 +226,13 @@ type EditPipeline struct { PipelineId string `json:"pipeline_id,omitempty" url:"-"` // Restart window of this pipeline. RestartWindow *RestartWindow `json:"restart_window,omitempty"` + // Write-only setting, available only in Create/Update calls. Specifies the + // user or service principal that the pipeline runs as. If not specified, + // the pipeline runs as the user who created the pipeline. + // + // Only `user_name` or `service_principal_name` can be specified. If both + // are specified, an error is thrown. + RunAs *RunAs `json:"run_as,omitempty"` // The default schema (database) where tables are read from or published to. // The presence of this field implies that the pipeline is in direct // publishing mode. @@ -1376,6 +1390,31 @@ func (f *RestartWindowDaysOfWeek) Type() string { return "RestartWindowDaysOfWeek" } +// Write-only setting, available only in Create/Update calls. Specifies the user +// or service principal that the pipeline runs as. If not specified, the +// pipeline runs as the user who created the pipeline. +// +// Only `user_name` or `service_principal_name` can be specified. If both are +// specified, an error is thrown. +type RunAs struct { + // Application ID of an active service principal. Setting this field + // requires the `servicePrincipal/user` role. + ServicePrincipalName string `json:"service_principal_name,omitempty"` + // The email of an active workspace user. Users can only set this field to + // their own email. + UserName string `json:"user_name,omitempty"` + + ForceSendFields []string `json:"-"` +} + +func (s *RunAs) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s RunAs) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type SchemaSpec struct { // Required. Destination catalog to store tables. DestinationCatalog string `json:"destination_catalog,omitempty"` diff --git a/service/pkg.go b/service/pkg.go index bd49b7d6..b891a954 100644 --- a/service/pkg.go +++ b/service/pkg.go @@ -48,10 +48,10 @@ // // - [marketplace.ConsumerProvidersAPI]: Providers are the entities that publish listings to the Marketplace. // -// - [catalog.CredentialsAPI]: A credential represents an authentication and authorization mechanism for accessing services on your cloud tenant. -// // - [provisioning.CredentialsAPI]: These APIs manage credential configurations for this workspace. // +// - [catalog.CredentialsAPI]: A credential represents an authentication and authorization mechanism for accessing services on your cloud tenant. +// // - [settings.CredentialsManagerAPI]: Credentials manager interacts with with Identity Providers to to perform token exchanges using stored credentials and refresh tokens. // // - [settings.CspEnablementAccountAPI]: The compliance security profile settings at the account level control whether to enable it for new workspaces. @@ -323,8 +323,8 @@ var ( _ *marketplace.ConsumerListingsAPI = nil _ *marketplace.ConsumerPersonalizationRequestsAPI = nil _ *marketplace.ConsumerProvidersAPI = nil - _ *catalog.CredentialsAPI = nil _ *provisioning.CredentialsAPI = nil + _ *catalog.CredentialsAPI = nil _ *settings.CredentialsManagerAPI = nil _ *settings.CspEnablementAccountAPI = nil _ *iam.CurrentUserAPI = nil diff --git a/service/serving/api.go b/service/serving/api.go index cddbdd73..5a2606d6 100755 --- a/service/serving/api.go +++ b/service/serving/api.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/retries" - "github.com/databricks/databricks-sdk-go/service/oauth2" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -468,7 +467,7 @@ func NewServingEndpointsDataPlane(client *client.DatabricksClient, return &ServingEndpointsDataPlaneAPI{ servingEndpointsDataPlaneImpl: servingEndpointsDataPlaneImpl{ client: client, - dataPlaneService: oauth2.NewDataPlaneService(), + dataPlaneService: NewDataPlaneService(), controlPlane: controlPlane, }, } diff --git a/service/oauth2/ext_data_plane.go b/service/serving/ext_data_plane.go similarity index 80% rename from service/oauth2/ext_data_plane.go rename to service/serving/ext_data_plane.go index aa9eeddc..37c80aed 100644 --- a/service/oauth2/ext_data_plane.go +++ b/service/serving/ext_data_plane.go @@ -1,27 +1,27 @@ -package oauth2 +package serving import ( "strings" "sync" - "golang.org/x/oauth2" + goauth "golang.org/x/oauth2" ) // DataPlaneService is an interface for services that access DataPlane. type DataPlaneService interface { - GetDataPlaneDetails(method string, params []string, refresh func(*DataPlaneInfo) (*oauth2.Token, error), infoGetter func() (*DataPlaneInfo, error)) (string, *oauth2.Token, error) + GetDataPlaneDetails(method string, params []string, refresh func(*DataPlaneInfo) (*goauth.Token, error), infoGetter func() (*DataPlaneInfo, error)) (string, *goauth.Token, error) } func NewDataPlaneService() DataPlaneService { return &dataPlaneServiceImpl{ infos: make(map[string]*DataPlaneInfo), - tokens: make(map[string]*oauth2.Token), + tokens: make(map[string]*goauth.Token), } } type dataPlaneServiceImpl struct { infos map[string]*DataPlaneInfo - tokens map[string]*oauth2.Token + tokens map[string]*goauth.Token // This class can be shared across multiple threads. // This mutex is used to synchronize access to the infos and tokens maps. mu sync.Mutex @@ -29,7 +29,7 @@ type dataPlaneServiceImpl struct { // GetDataPlaneDetails returns the endpoint URL and token. It returns a cached token if it is valid, // otherwise it refreshes the token and returns the new token. -func (o *dataPlaneServiceImpl) GetDataPlaneDetails(method string, params []string, refresh func(*DataPlaneInfo) (*oauth2.Token, error), infoGetter func() (*DataPlaneInfo, error)) (string, *oauth2.Token, error) { +func (o *dataPlaneServiceImpl) GetDataPlaneDetails(method string, params []string, refresh func(*DataPlaneInfo) (*goauth.Token, error), infoGetter func() (*DataPlaneInfo, error)) (string, *goauth.Token, error) { key := o.generateKey(method, params) info, err := o.getInfo(key, infoGetter) if err != nil { @@ -60,7 +60,7 @@ func (o *dataPlaneServiceImpl) getInfo(key string, infoGetter func() (*DataPlane return info, nil } -func (o *dataPlaneServiceImpl) refreshToken(key string, info *DataPlaneInfo, refresh func(*DataPlaneInfo) (*oauth2.Token, error)) (*oauth2.Token, error) { +func (o *dataPlaneServiceImpl) refreshToken(key string, info *DataPlaneInfo, refresh func(*DataPlaneInfo) (*goauth.Token, error)) (*goauth.Token, error) { token, tokenOk := o.tokens[key] if !tokenOk || !token.Valid() { o.mu.Lock() diff --git a/service/oauth2/ext_data_plane_test.go b/service/serving/ext_data_plane_test.go similarity index 89% rename from service/oauth2/ext_data_plane_test.go rename to service/serving/ext_data_plane_test.go index ca622a45..27a0972a 100644 --- a/service/oauth2/ext_data_plane_test.go +++ b/service/serving/ext_data_plane_test.go @@ -1,11 +1,11 @@ -package oauth2 +package serving import ( "testing" "time" "github.com/stretchr/testify/assert" - "golang.org/x/oauth2" + goauth "golang.org/x/oauth2" ) type infoMock struct { @@ -22,11 +22,11 @@ func (i *infoMock) DataPlaneInfoGetter() (*DataPlaneInfo, error) { type tokenRefreshSpy struct { called bool expectedInfo *DataPlaneInfo - token *oauth2.Token + token *goauth.Token err error } -func (t *tokenRefreshSpy) TokenRefresh(info *DataPlaneInfo) (*oauth2.Token, error) { +func (t *tokenRefreshSpy) TokenRefresh(info *DataPlaneInfo) (*goauth.Token, error) { t.expectedInfo = info t.called = true return t.token, t.err @@ -41,7 +41,7 @@ func TestTokenNotCached(t *testing.T) { err: nil, } s := tokenRefreshSpy{ - token: &oauth2.Token{ + token: &goauth.Token{ AccessToken: "token", TokenType: "type", Expiry: time.Now().Add(1 * time.Hour), @@ -50,7 +50,7 @@ func TestTokenNotCached(t *testing.T) { } c := dataPlaneServiceImpl{ infos: make(map[string]*DataPlaneInfo), - tokens: make(map[string]*oauth2.Token), + tokens: make(map[string]*goauth.Token), } url, token, err := c.GetDataPlaneDetails("method", []string{"params"}, s.TokenRefresh, info.DataPlaneInfoGetter) assert.NoError(t, err) @@ -71,7 +71,7 @@ func TestTokenCached(t *testing.T) { err: nil, } s := tokenRefreshSpy{ - token: &oauth2.Token{ + token: &goauth.Token{ AccessToken: "token", TokenType: "type", Expiry: time.Now().Add(1 * time.Hour), @@ -80,7 +80,7 @@ func TestTokenCached(t *testing.T) { } c := dataPlaneServiceImpl{} c.infos = make(map[string]*DataPlaneInfo) - c.tokens = make(map[string]*oauth2.Token) + c.tokens = make(map[string]*goauth.Token) c.infos["method/params"] = info.info c.tokens["method/params"] = s.token url, token, err := c.GetDataPlaneDetails("method", []string{"params"}, s.TokenRefresh, info.DataPlaneInfoGetter) @@ -102,13 +102,13 @@ func TestTokenExpired(t *testing.T) { err: nil, } - expired := &oauth2.Token{ + expired := &goauth.Token{ AccessToken: "oldToken", TokenType: "type", Expiry: time.Now().Add(-1 * time.Hour), } s := tokenRefreshSpy{ - token: &oauth2.Token{ + token: &goauth.Token{ AccessToken: "token", TokenType: "type", Expiry: time.Now().Add(1 * time.Hour), @@ -117,7 +117,7 @@ func TestTokenExpired(t *testing.T) { } c := dataPlaneServiceImpl{} c.infos = make(map[string]*DataPlaneInfo) - c.tokens = make(map[string]*oauth2.Token) + c.tokens = make(map[string]*goauth.Token) c.infos["method/params"] = info.info c.tokens["method/params"] = expired url, token, err := c.GetDataPlaneDetails("method", []string{"params"}, s.TokenRefresh, info.DataPlaneInfoGetter) @@ -138,7 +138,7 @@ func TestTokenInfoError(t *testing.T) { s := tokenRefreshSpy{} c := dataPlaneServiceImpl{ infos: make(map[string]*DataPlaneInfo), - tokens: make(map[string]*oauth2.Token), + tokens: make(map[string]*goauth.Token), } url, token, err := c.GetDataPlaneDetails("method", []string{"params"}, s.TokenRefresh, info.DataPlaneInfoGetter) assert.ErrorIs(t, err, assert.AnError) @@ -162,7 +162,7 @@ func TestTokenRefreshError(t *testing.T) { } c := dataPlaneServiceImpl{ infos: make(map[string]*DataPlaneInfo), - tokens: make(map[string]*oauth2.Token), + tokens: make(map[string]*goauth.Token), } url, token, err := c.GetDataPlaneDetails("method", []string{"params"}, s.TokenRefresh, info.DataPlaneInfoGetter) assert.ErrorIs(t, err, assert.AnError) diff --git a/service/serving/impl.go b/service/serving/impl.go index 0b211b21..77476ef7 100755 --- a/service/serving/impl.go +++ b/service/serving/impl.go @@ -11,8 +11,6 @@ import ( "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/httpclient" goauth "golang.org/x/oauth2" - - "github.com/databricks/databricks-sdk-go/service/oauth2" ) // unexported type that holds implementations of just ServingEndpoints API methods @@ -183,7 +181,7 @@ func (a *servingEndpointsImpl) UpdatePermissions(ctx context.Context, request Se // unexported type that holds implementations of just ServingEndpointsDataPlane API methods type servingEndpointsDataPlaneImpl struct { - dataPlaneService oauth2.DataPlaneService + dataPlaneService DataPlaneService controlPlane *ServingEndpointsAPI client *client.DatabricksClient } @@ -196,7 +194,7 @@ func (a *servingEndpointsDataPlaneImpl) Query(ctx context.Context, request Query if err != nil { return nil, err } - infoGetter := func() (*oauth2.DataPlaneInfo, error) { + infoGetter := func() (*DataPlaneInfo, error) { response, err := a.controlPlane.Get(ctx, getRequest) if err != nil { return nil, err @@ -206,7 +204,7 @@ func (a *servingEndpointsDataPlaneImpl) Query(ctx context.Context, request Query } return response.DataPlaneInfo.QueryInfo, nil } - refresh := func(info *oauth2.DataPlaneInfo) (*goauth.Token, error) { + refresh := func(info *DataPlaneInfo) (*goauth.Token, error) { return a.client.GetOAuthToken(ctx, info.AuthorizationDetails, token) } getParams := []string{ diff --git a/service/serving/model.go b/service/serving/model.go index 6ac7eb49..ade65278 100755 --- a/service/serving/model.go +++ b/service/serving/model.go @@ -7,7 +7,6 @@ import ( "io" "github.com/databricks/databricks-sdk-go/marshal" - "github.com/databricks/databricks-sdk-go/service/oauth2" ) type Ai21LabsConfig struct { @@ -498,6 +497,23 @@ func (s CreateServingEndpoint) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } +type DataPlaneInfo struct { + // Authorization details as a string. + AuthorizationDetails string `json:"authorization_details,omitempty"` + // The URL of the endpoint for this operation in the dataplane. + EndpointUrl string `json:"endpoint_url,omitempty"` + + ForceSendFields []string `json:"-"` +} + +func (s *DataPlaneInfo) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s DataPlaneInfo) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type DatabricksModelServingConfig struct { // The Databricks secret key reference for a Databricks API token that // corresponds to a user or service principal with Can Query access to the @@ -981,7 +997,7 @@ type LogsRequest struct { type ModelDataPlaneInfo struct { // Information required to query DataPlane API 'query' endpoint. - QueryInfo *oauth2.DataPlaneInfo `json:"query_info,omitempty"` + QueryInfo *DataPlaneInfo `json:"query_info,omitempty"` } type OpenAiConfig struct { From 23d9c1ea2bf131c426b666573ecac4d798578bb4 Mon Sep 17 00:00:00 2001 From: Renaud Hartert Date: Tue, 7 Jan 2025 15:57:39 +0100 Subject: [PATCH 4/4] [Internal] Move package credentials in config (#1115) ## What changes are proposed in this pull request? This PR moves the `credentials` package in `config`, thus centralizing the credentials logic in a single place. The goal of this change is to prepare the current SDK-Beta to facilitate migration to the new SDK that is currently being developed. This PR introduces a breaking change by changing the import paths to `CredentialsProvider`. However, we believe that the impact is fairly contained as (i) `CredentialsProvider` is mostly internally, and (ii) the breaking change can be addressed by updating the import path. ## How is this tested? Through normal unit test and integration tests. --- config/api_client.go | 2 +- config/auth_azure_cli.go | 2 +- config/auth_azure_client_secret.go | 2 +- config/auth_azure_github_oidc.go | 2 +- config/auth_azure_msi.go | 2 +- config/auth_basic.go | 2 +- config/auth_databricks_cli.go | 2 +- config/auth_default.go | 2 +- config/auth_gcp_google_credentials.go | 2 +- config/auth_gcp_google_id.go | 2 +- config/auth_m2m.go | 2 +- config/auth_metadata_service.go | 2 +- config/auth_pat.go | 2 +- config/config.go | 2 +- config/credentials/credentials.go | 68 +++++++++++++++++++++++ credentials/credentials_provider.go | 26 --------- credentials/oauth_credentials_provider.go | 34 ------------ credentials/oauth_token.go | 14 ----- examples/custom-auth/main.go | 2 +- httpclient/oauth_token.go | 2 +- 20 files changed, 84 insertions(+), 90 deletions(-) create mode 100644 config/credentials/credentials.go delete mode 100644 credentials/credentials_provider.go delete mode 100644 credentials/oauth_credentials_provider.go delete mode 100644 credentials/oauth_token.go diff --git a/config/api_client.go b/config/api_client.go index 0913e35b..0101dc2a 100644 --- a/config/api_client.go +++ b/config/api_client.go @@ -9,7 +9,7 @@ import ( "time" "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/useragent" ) diff --git a/config/auth_azure_cli.go b/config/auth_azure_cli.go index 802ace3f..56959f4c 100644 --- a/config/auth_azure_cli.go +++ b/config/auth_azure_cli.go @@ -11,7 +11,7 @@ import ( "golang.org/x/oauth2" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/logger" ) diff --git a/config/auth_azure_client_secret.go b/config/auth_azure_client_secret.go index 1ba33e37..85f09bb2 100644 --- a/config/auth_azure_client_secret.go +++ b/config/auth_azure_client_secret.go @@ -8,7 +8,7 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/logger" ) diff --git a/config/auth_azure_github_oidc.go b/config/auth_azure_github_oidc.go index fdb106a8..2f82214f 100644 --- a/config/auth_azure_github_oidc.go +++ b/config/auth_azure_github_oidc.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2" diff --git a/config/auth_azure_msi.go b/config/auth_azure_msi.go index 68ab5044..7a947d7d 100644 --- a/config/auth_azure_msi.go +++ b/config/auth_azure_msi.go @@ -8,7 +8,7 @@ import ( "net/http" "time" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2" diff --git a/config/auth_basic.go b/config/auth_basic.go index 4374bf4e..8b25fa45 100644 --- a/config/auth_basic.go +++ b/config/auth_basic.go @@ -6,7 +6,7 @@ import ( "fmt" "net/http" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" ) type BasicCredentials struct { diff --git a/config/auth_databricks_cli.go b/config/auth_databricks_cli.go index 7f054d2e..1ec79830 100644 --- a/config/auth_databricks_cli.go +++ b/config/auth_databricks_cli.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2" ) diff --git a/config/auth_default.go b/config/auth_default.go index 9a6a3cf2..5fc3080f 100644 --- a/config/auth_default.go +++ b/config/auth_default.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/logger" ) diff --git a/config/auth_gcp_google_credentials.go b/config/auth_gcp_google_credentials.go index 317eb6be..127a590f 100644 --- a/config/auth_gcp_google_credentials.go +++ b/config/auth_gcp_google_credentials.go @@ -6,7 +6,7 @@ import ( "io/ioutil" "os" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2/google" "google.golang.org/api/idtoken" diff --git a/config/auth_gcp_google_id.go b/config/auth_gcp_google_id.go index 4dd291c1..3d3a600b 100644 --- a/config/auth_gcp_google_id.go +++ b/config/auth_gcp_google_id.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2" "google.golang.org/api/impersonate" diff --git a/config/auth_m2m.go b/config/auth_m2m.go index 5399228f..70fe07d9 100644 --- a/config/auth_m2m.go +++ b/config/auth_m2m.go @@ -8,7 +8,7 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/logger" ) diff --git a/config/auth_metadata_service.go b/config/auth_metadata_service.go index 3f349128..beae500c 100644 --- a/config/auth_metadata_service.go +++ b/config/auth_metadata_service.go @@ -8,7 +8,7 @@ import ( "net/url" "time" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2" diff --git a/config/auth_pat.go b/config/auth_pat.go index 72ffadb3..dc51256c 100644 --- a/config/auth_pat.go +++ b/config/auth_pat.go @@ -5,7 +5,7 @@ import ( "fmt" "net/http" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" ) type PatCredentials struct { diff --git a/config/config.go b/config/config.go index fcf69d2c..b451d3ec 100644 --- a/config/config.go +++ b/config/config.go @@ -13,7 +13,7 @@ import ( "github.com/databricks/databricks-sdk-go/common" "github.com/databricks/databricks-sdk-go/common/environment" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2" diff --git a/config/credentials/credentials.go b/config/credentials/credentials.go new file mode 100644 index 00000000..fe1041f1 --- /dev/null +++ b/config/credentials/credentials.go @@ -0,0 +1,68 @@ +package credentials + +import ( + "net/http" + + "golang.org/x/oauth2" +) + +// CredentialsProvider is an interface for providing credentials to the client. +// Implementations of this interface should set the necessary headers on the request. +type CredentialsProvider interface { + // SetHeaders sets the necessary headers on the request. + SetHeaders(r *http.Request) error +} + +type credentialsProvider struct { + setHeaders func(r *http.Request) error +} + +func (c *credentialsProvider) SetHeaders(r *http.Request) error { + return c.setHeaders(r) +} + +func NewCredentialsProvider(visitor func(r *http.Request) error) CredentialsProvider { + return &credentialsProvider{ + setHeaders: visitor, + } +} + +// OAuthCredentialsProvider is a specialized CredentialsProvider uses and provides an OAuth token. +type OAuthCredentialsProvider interface { + CredentialsProvider + // Token returns the OAuth token generated by the provider. + Token() (*oauth2.Token, error) +} + +type oauthCredentialsProvider struct { + setHeaders func(r *http.Request) error + token func() (*oauth2.Token, error) +} + +func (c *oauthCredentialsProvider) SetHeaders(r *http.Request) error { + return c.setHeaders(r) +} + +func (c *oauthCredentialsProvider) Token() (*oauth2.Token, error) { + return c.token() +} + +func NewOAuthCredentialsProvider(visitor func(r *http.Request) error, tokenProvider func() (*oauth2.Token, error)) OAuthCredentialsProvider { + return &oauthCredentialsProvider{ + setHeaders: visitor, + token: tokenProvider, + } +} + +// OAuthToken represents an OAuth token as defined by the OAuth 2.0 Authorization Framework. +// https://datatracker.ietf.org/doc/html/rfc6749 +type OAuthToken struct { + // The access token issued by the authorization server. This is the token that will be used to authenticate requests. + AccessToken string `json:"access_token" auth:",sensitive"` + // Time in seconds until the token expires. + ExpiresIn int `json:"expires_in"` + // The scope of the token. This is a space-separated list of strings that represent the permissions granted by the token. + Scope string `json:"scope"` + // The type of token that was issued. + TokenType string `json:"token_type"` +} diff --git a/credentials/credentials_provider.go b/credentials/credentials_provider.go deleted file mode 100644 index 4e82ab3b..00000000 --- a/credentials/credentials_provider.go +++ /dev/null @@ -1,26 +0,0 @@ -package credentials - -import ( - "net/http" -) - -// CredentialsProvider is an interface for providing credentials to the client. -// Implementations of this interface should set the necessary headers on the request. -type CredentialsProvider interface { - // SetHeaders sets the necessary headers on the request. - SetHeaders(r *http.Request) error -} - -type credentialsProvider struct { - setHeaders func(r *http.Request) error -} - -func (c *credentialsProvider) SetHeaders(r *http.Request) error { - return c.setHeaders(r) -} - -func NewCredentialsProvider(visitor func(r *http.Request) error) CredentialsProvider { - return &credentialsProvider{ - setHeaders: visitor, - } -} diff --git a/credentials/oauth_credentials_provider.go b/credentials/oauth_credentials_provider.go deleted file mode 100644 index fe6d2dd0..00000000 --- a/credentials/oauth_credentials_provider.go +++ /dev/null @@ -1,34 +0,0 @@ -package credentials - -import ( - "net/http" - - "golang.org/x/oauth2" -) - -// OAuthCredentialsProvider is a specialized CredentialsProvider uses and provides an OAuth token. -type OAuthCredentialsProvider interface { - CredentialsProvider - // Token returns the OAuth token generated by the provider. - Token() (*oauth2.Token, error) -} - -type oauthCredentialsProvider struct { - setHeaders func(r *http.Request) error - token func() (*oauth2.Token, error) -} - -func (c *oauthCredentialsProvider) SetHeaders(r *http.Request) error { - return c.setHeaders(r) -} - -func (c *oauthCredentialsProvider) Token() (*oauth2.Token, error) { - return c.token() -} - -func NewOAuthCredentialsProvider(visitor func(r *http.Request) error, tokenProvider func() (*oauth2.Token, error)) OAuthCredentialsProvider { - return &oauthCredentialsProvider{ - setHeaders: visitor, - token: tokenProvider, - } -} diff --git a/credentials/oauth_token.go b/credentials/oauth_token.go deleted file mode 100644 index a1f6c131..00000000 --- a/credentials/oauth_token.go +++ /dev/null @@ -1,14 +0,0 @@ -package credentials - -// OAuthToken represents an OAuth token as defined by the OAuth 2.0 Authorization Framework. -// https://datatracker.ietf.org/doc/html/rfc6749 -type OAuthToken struct { - // The access token issued by the authorization server. This is the token that will be used to authenticate requests. - AccessToken string `json:"access_token" auth:",sensitive"` - // Time in seconds until the token expires. - ExpiresIn int `json:"expires_in"` - // The scope of the token. This is a space-separated list of strings that represent the permissions granted by the token. - Scope string `json:"scope"` - // The type of token that was issued. - TokenType string `json:"token_type"` -} diff --git a/examples/custom-auth/main.go b/examples/custom-auth/main.go index c0eb779f..76b48cab 100644 --- a/examples/custom-auth/main.go +++ b/examples/custom-auth/main.go @@ -10,7 +10,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "github.com/databricks/databricks-sdk-go/service/compute" ) diff --git a/httpclient/oauth_token.go b/httpclient/oauth_token.go index 142afd48..cb6ad5cc 100644 --- a/httpclient/oauth_token.go +++ b/httpclient/oauth_token.go @@ -5,7 +5,7 @@ import ( "net/http" "time" - "github.com/databricks/databricks-sdk-go/credentials" + "github.com/databricks/databricks-sdk-go/config/credentials" "golang.org/x/oauth2" )