From a5dbc47d3565b01b59c03481a51b729e4dcb5460 Mon Sep 17 00:00:00 2001 From: Marcos Gaeta Date: Fri, 19 Jul 2024 15:19:00 -0700 Subject: [PATCH 1/6] Add spaces resources --- pkg/connector/client/confluence.go | 330 +++++++++++++++++++++++++++++ pkg/connector/client/models.go | 80 +++++++ pkg/connector/connector.go | 6 + pkg/connector/spaces.go | 244 +++++++++++++++++++++ pkg/connector/spaces_test.go | 64 ++++++ 5 files changed, 724 insertions(+) create mode 100644 pkg/connector/spaces.go create mode 100644 pkg/connector/spaces_test.go diff --git a/pkg/connector/client/confluence.go b/pkg/connector/client/confluence.go index 79efd3b9..a56c3d9b 100644 --- a/pkg/connector/client/confluence.go +++ b/pkg/connector/client/confluence.go @@ -15,6 +15,8 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/helpers" "github.com/conductorone/baton-sdk/pkg/uhttp" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -351,6 +353,334 @@ func strToInt(s string) int { return i } +// GetSpaces uses pagination to get a list of spaces from the global list. +func (c *ConfluenceClient) GetSpaces( + ctx context.Context, + pageSize int, + paginationCursor string, +) ( + []ConfluenceSpace, + string, + *v2.RateLimitDescription, + error, +) { + spacesListUrl, err := c.genURLWithPaginationCursor( + SpacesListUrlPath, + pageSize, + paginationCursor, + ) + if err != nil { + return nil, "", nil, err + } + + var response *confluenceSpaceList + ratelimitData, err := c.get(ctx, spacesListUrl, &response) + if err != nil { + return nil, "", ratelimitData, err + } + + cursor := extractPaginationCursor(response.Links) + spaces := response.Results + + return spaces, cursor, ratelimitData, nil +} + +func (c *ConfluenceClient) ConfluenceSpaceOperations( + ctx context.Context, + cursor string, + pageSize int, + spaceId string, +) ( + []ConfluenceSpaceOperation, + string, + *v2.RateLimitDescription, + error, +) { + logger := ctxzap.Extract(ctx) + logger.Debug("fetching space", zap.String("spaceId", spaceId)) + + spaceUrl, err := c.genURLWithPaginationCursor( + fmt.Sprintf(spacesGetUrlPath+"?include-operations=1", spaceId), + pageSize, + cursor, + ) + + if err != nil { + return nil, "", nil, err + } + + var response *ConfluenceSpace + ratelimitData, err := c.get(ctx, spaceUrl, &response) + if err != nil { + return nil, "", ratelimitData, err + } + + operations := make([]ConfluenceSpaceOperation, 0) + operations = append(operations, response.Operations.Results...) + + nextToken := "" + if response.Operations.Meta.HasMore { + nextToken = response.Operations.Meta.Cursor + } + + return operations, nextToken, ratelimitData, nil +} + +func (c *ConfluenceClient) GetSpacePermissions( + ctx context.Context, + pageToken string, + pageSize int, + spaceId string, +) ( + []ConfluenceSpacePermission, + string, + *v2.RateLimitDescription, + error, +) { + spacePermissionsListUrl, err := c.genURLWithPaginationCursor( + fmt.Sprintf(SpacePermissionsListUrlPath, spaceId), + pageSize, + pageToken, + ) + if err != nil { + return nil, "", nil, err + } + + var response *ConfluenceSpacePermissionResponse + ratelimitData, err := c.get( + ctx, + spacePermissionsListUrl, + &response, + ) + if err != nil { + return nil, "", ratelimitData, err + } + cursor := extractPaginationCursor(response.Links) + permissions := make([]ConfluenceSpacePermission, 0) + permissions = append(permissions, response.Results...) + + return permissions, cursor, ratelimitData, nil +} + +// getSubjectTypeFromPrincipalType map between ConductorOne representation and +// Confluence representation. It just so happens that the representations are +// the same, but I don't want to pass it straight along in case we get new +// principal types that aren't a 100% match. +func getSubjectTypeFromPrincipalType(principalType string) (string, error) { + switch principalType { + case "user": + return "user", nil + case "group": + return "group", nil + } + return "", fmt.Errorf("principal type '%s' is not supported", principalType) +} + +func (c *ConfluenceClient) AddSpacePermission( + ctx context.Context, + spaceName string, + key string, + target string, + principalId string, + principalType string, +) ( + *v2.RateLimitDescription, + error, +) { + spacePermissionsListUrl, err := c.genURLNonPaginated( + fmt.Sprintf(spacePermissionsCreateUrlPath, spaceName), + ) + if err != nil { + return nil, err + } + + subjectType, err := getSubjectTypeFromPrincipalType(principalType) + if err != nil { + return nil, err + } + + bodyBytes, err := json.Marshal( + CreateSpacePermissionRequestBody{ + SpacePermissionSubject{ + Identifier: principalId, + Type: subjectType, + }, + SpacePermissionOperation{ + Key: key, + Target: target, + }, + }, + ) + if err != nil { + return nil, err + } + + body := strings.NewReader(string(bodyBytes)) + + var response bool + ratelimitData, err := c.post( + ctx, + spacePermissionsListUrl, + &response, + body, + ) + if err != nil { + return ratelimitData, err + } + + return ratelimitData, nil +} + +// findSpacePermission - There isn't a way to look up a permission by these +// fields, so we need to list _all_ permissions in order to find the permission. +func (c *ConfluenceClient) findSpacePermission( + ctx context.Context, + spaceId string, + key string, + target string, + principalId string, + principalType string, +) ( + *ConfluenceSpacePermission, + *v2.RateLimitDescription, + error, +) { + // We need to list _all_ permissions in order to figure out the permission's ID. + cursor := "" + for { + listPermissionsUrl, err := c.genURLWithPaginationCursor( + fmt.Sprintf( + SpacePermissionsListUrlPath, + spaceId, + ), + maxResults, + cursor, + ) + if err != nil { + return nil, nil, err + } + + var response *ConfluenceSpacePermissionResponse + ratelimitData, err := c.get( + ctx, + listPermissionsUrl, + &response, + ) + if err != nil { + return nil, ratelimitData, err + } + for _, permission := range response.Results { + if permission.Principal.Id == principalId && + permission.Principal.Type == principalType && + permission.Operation.Key == key && + permission.Operation.TargetType == target { + return &permission, nil, nil + } + } + cursor = extractPaginationCursor(response.Links) + if cursor == "" { + break + } + } + + return nil, nil, fmt.Errorf("space permission not found") +} + +// findSpace - The v1 and v2 API are slightly different. The former uses "space +// key", which is like the URL slug for the space. The latter use plain ID. +func (c *ConfluenceClient) findSpace( + ctx context.Context, + spaceId string, +) ( + *ConfluenceSpace, + *v2.RateLimitDescription, + error, +) { + getSpaceUrl, err := c.genURLNonPaginated( + fmt.Sprintf( + spacesGetUrlPath, + spaceId, + ), + ) + if err != nil { + return nil, nil, err + } + + var response *ConfluenceSpace + ratelimitData, err := c.get( + ctx, + getSpaceUrl, + &response, + ) + if err != nil { + return nil, ratelimitData, err + } + return response, ratelimitData, nil +} + +func (c *ConfluenceClient) RemoveSpacePermission( + ctx context.Context, + spaceId string, + key string, + target string, + principalId string, + principalType string, +) ( + *v2.RateLimitDescription, + error, +) { + permission, ratelimitData, err := c.findSpacePermission( + ctx, + spaceId, + key, + target, + principalId, + principalType, + ) + + if err != nil { + return ratelimitData, err + } + + space, ratelimitData, err := c.findSpace(ctx, spaceId) + if err != nil { + return ratelimitData, err + } + + deletePermissionUrl, err := c.genURLNonPaginated( + fmt.Sprintf( + spacePermissionsUpdateUrlPath, + space.Key, + permission.Id, + ), + ) + if err != nil { + return nil, err + } + + var response bool + ratelimitData, err = c.delete( + ctx, + deletePermissionUrl, + &response, + ) + if err != nil { + return ratelimitData, err + } + + return ratelimitData, nil +} + +// extractPaginationCursor returns the query parameters from the "next" link in +// the list response. +func extractPaginationCursor(links ConfluenceLink) string { + parsedUrl, err := url.Parse(links.Next) + if err != nil { + return "" + } + return parsedUrl.Query().Get("cursor") +} + // WithConfluenceRatelimitData Per the docs: transient 5XX errors should be // treated as 429/too-many-requests if they have a retry header. 503 errors were // the only ones explicitly called out, but I guess it's possible for others too diff --git a/pkg/connector/client/models.go b/pkg/connector/client/models.go index a14d8279..a8d4feb6 100644 --- a/pkg/connector/client/models.go +++ b/pkg/connector/client/models.go @@ -55,6 +55,86 @@ type confluenceGroupList struct { Results []ConfluenceGroup `json:"results"` } +type ConfluenceSpaceDescriptionValue struct { + Value string `json:"value"` + Representation string `json:"representation"` +} + +type ConfluenceSpaceDescription struct { + Plain ConfluenceSpaceDescriptionValue `json:"plain"` +} + +type ConfluenceMeta struct { + HasMore bool `json:"hasMore"` + Cursor string `json:"cursor"` +} + +type ConfluenceSpaceOperationsResponse struct { + Links ConfluenceLink `json:"_links"` + Meta ConfluenceMeta `json:"meta"` + Results []ConfluenceSpaceOperation `json:"results"` +} + +type ConfluenceSpacePermissionResponse struct { + Links ConfluenceLink `json:"_links"` + Results []ConfluenceSpacePermission `json:"results"` +} + +type ConfluenceSpacePermission struct { + Id string `json:"id"` + Principal ConfluenceSpacePermissionPrincipal `json:"principal"` + Operation ConfluenceSpacePermissionOperation `json:"operation"` +} + +type ConfluenceSpacePermissionPrincipal struct { + Id string `json:"id"` + Type string `json:"type"` +} + +type ConfluenceSpaceOperation struct { + Operation string `json:"operation"` + TargetType string `json:"targetType"` +} + +type ConfluenceSpacePermissionOperation struct { + Key string `json:"key"` + TargetType string `json:"targetType"` +} + +type ConfluenceSpace struct { + AuthorId string `json:"authorId"` + CreatedAt string `json:"createdAt"` + Description ConfluenceSpaceDescription `json:"description"` + HomepageId string `json:"homepageId"` + Icon string `json:"icon"` + Id string `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Operations ConfluenceSpaceOperationsResponse `json:"operations"` + Status string `json:"status"` + Type string `json:"type"` +} + +type confluenceSpaceList struct { + Links ConfluenceLink `json:"_links"` + Results []ConfluenceSpace `json:"results"` +} + +type SpacePermissionSubject struct { + Type string `json:"type"` + Identifier string `json:"identifier"` +} + +type SpacePermissionOperation struct { + Key string `json:"key"` + Target string `json:"target"` +} + +type CreateSpacePermissionRequestBody struct { + Subject SpacePermissionSubject `json:"subject"` + Operation SpacePermissionOperation `json:"operation"` +} + type AddUserToGroupRequestBody struct { AccountId string `json:"accountId"` } diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 88f071da..96ce09c0 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -30,6 +30,11 @@ var ( }, Annotations: annotationsForUserResourceType(), } + spaceResourceType = &v2.ResourceType{ + Id: "space", + DisplayName: "Space", + Traits: []v2.ResourceType_Trait{}, + } ) type Config struct { @@ -94,5 +99,6 @@ func (c *Confluence) ResourceSyncers(ctx context.Context) []connectorbuilder.Res return []connectorbuilder.ResourceSyncer{ groupBuilder(c.client), userBuilder(c.client), + newSpaceBuilder(c.client), } } diff --git a/pkg/connector/spaces.go b/pkg/connector/spaces.go new file mode 100644 index 00000000..604d5395 --- /dev/null +++ b/pkg/connector/spaces.go @@ -0,0 +1,244 @@ +package connector + +import ( + "context" + "fmt" + "strings" + + "github.com/conductorone/baton-confluence/pkg/connector/client" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" +) + +const separator = "-" + +func CreateEntitlementName(operation client.ConfluenceSpaceOperation) string { + return fmt.Sprintf( + "%s%s%s", + operation.Operation, + separator, + operation.TargetType, + ) +} + +// GetEntitlementComponents returns the operation and target in that order. +func GetEntitlementComponents(operation string) (string, string) { + parts := strings.Split(operation, separator) + return parts[0], parts[1] +} + +type spaceBuilder struct { + client client.ConfluenceClient +} + +func (o *spaceBuilder) ResourceType(ctx context.Context) *v2.ResourceType { + return spaceResourceType +} + +// List returns all the spaces from the database as resource objects. +func (o *spaceBuilder) List( + ctx context.Context, + parentResourceID *v2.ResourceId, + pToken *pagination.Token, +) ( + []*v2.Resource, + string, + annotations.Annotations, + error, +) { + spaces, nextToken, ratelimitData, err := o.client.GetSpaces( + ctx, + ResourcesPageSize, + pToken.Token, + ) + outputAnnotations := WithRateLimitAnnotations(ratelimitData) + if err != nil { + return nil, "", outputAnnotations, err + } + rv := make([]*v2.Resource, 0) + for _, space := range spaces { + spaceCopy := space + ur, err := spaceResource(ctx, &spaceCopy) + if err != nil { + return nil, "", nil, err + } + + rv = append(rv, ur) + } + + return rv, nextToken, outputAnnotations, nil +} + +func (o *spaceBuilder) Entitlements( + ctx context.Context, + resource *v2.Resource, + pToken *pagination.Token, +) ( + []*v2.Entitlement, + string, + annotations.Annotations, + error, +) { + logger := ctxzap.Extract(ctx) + logger.Debug( + "Starting call to Spaces.Entitlements", + zap.String("resource.DisplayName", resource.DisplayName), + zap.String("resource.Id.Resource", resource.Id.Resource), + ) + entitlements := make([]*v2.Entitlement, 0) + spacePermissions, nextToken, ratelimitData, err := o.client.ConfluenceSpaceOperations( + ctx, + pToken.Token, + pToken.Size, + resource.Id.Resource, + ) + outputAnnotations := WithRateLimitAnnotations(ratelimitData) + if err != nil { + return nil, "", outputAnnotations, err + } + + for _, operation := range spacePermissions { + operationName := CreateEntitlementName(operation) + entitlements = append( + entitlements, + entitlement.NewPermissionEntitlement( + resource, + operationName, + entitlement.WithGrantableTo(resourceTypeUser), + entitlement.WithGrantableTo(resourceTypeGroup), + entitlement.WithDisplayName( + fmt.Sprintf( + "Can %s %s", + operationName, + resource.DisplayName, + ), + ), + entitlement.WithDescription( + fmt.Sprintf( + "Has permission to %s the %s space in Confluence Data Center", + operationName, + resource.DisplayName, + ), + ), + )) + } + return entitlements, nextToken, outputAnnotations, nil +} + +// Grants the grants for a given space are the permissions. +func (o *spaceBuilder) Grants( + ctx context.Context, + resource *v2.Resource, + pToken *pagination.Token, +) ( + []*v2.Grant, + string, + annotations.Annotations, + error, +) { + permissionsList, nextToken, ratelimitData, err := o.client.GetSpacePermissions( + ctx, + pToken.Token, + pToken.Size, + resource.Id.Resource, + ) + outputAnnotations := WithRateLimitAnnotations(ratelimitData) + if err != nil { + return nil, "", outputAnnotations, err + } + + var permissions []*v2.Grant + for _, permission := range permissionsList { + var resourceType string + switch permission.Principal.Type { + case "user": + resourceType = resourceTypeUser.Id + case "group": + resourceType = resourceTypeGroup.Id + default: + // Skip if the type is "role". + continue + } + + permissionName := fmt.Sprintf( + "%s-%s", + permission.Operation.Key, + permission.Operation.TargetType, + ) + + permissions = append( + permissions, + grant.NewGrant( + resource, + permissionName, + &v2.ResourceId{ + ResourceType: resourceType, + Resource: permission.Principal.Id, + }, + )) + } + + return permissions, nextToken, outputAnnotations, nil +} + +func (o *spaceBuilder) Grant( + ctx context.Context, + principal *v2.Resource, + entitlement *v2.Entitlement, +) (annotations.Annotations, error) { + spaceName := entitlement.Resource.Id.Resource + key, target := GetEntitlementComponents(entitlement.Slug) + ratelimitData, err := o.client.AddSpacePermission( + ctx, + spaceName, + key, + target, + principal.Id.Resource, + principal.Id.ResourceType, + ) + outputAnnotations := WithRateLimitAnnotations(ratelimitData) + return outputAnnotations, err +} + +func (o *spaceBuilder) Revoke( + ctx context.Context, + grant *v2.Grant, +) (annotations.Annotations, error) { + spaceId := grant.Entitlement.Resource.Id.Resource + key, target := GetEntitlementComponents(grant.Entitlement.Slug) + ratelimitData, err := o.client.RemoveSpacePermission( + ctx, + spaceId, + key, + target, + grant.Principal.Id.Resource, + grant.Principal.Id.ResourceType, + ) + outputAnnotations := WithRateLimitAnnotations(ratelimitData) + return outputAnnotations, err +} + +func newSpaceBuilder(client *client.ConfluenceClient) *spaceBuilder { + return &spaceBuilder{ + client: *client, + } +} + +func spaceResource(ctx context.Context, space *client.ConfluenceSpace) (*v2.Resource, error) { + createdResource, err := resource.NewResource( + space.Name, + spaceResourceType, + space.Id, + ) + if err != nil { + return nil, err + } + + return createdResource, nil +} diff --git a/pkg/connector/spaces_test.go b/pkg/connector/spaces_test.go new file mode 100644 index 00000000..da82071e --- /dev/null +++ b/pkg/connector/spaces_test.go @@ -0,0 +1,64 @@ +package connector + +import ( + "context" + "testing" + + "github.com/conductorone/baton-confluence/pkg/connector/client" + "github.com/conductorone/baton-confluence/test" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/stretchr/testify/require" +) + +func TestSpaces(t *testing.T) { + ctx := context.Background() + server := test.FixturesServer() + defer server.Close() + + confluenceClient, err := client.NewConfluenceClient( + ctx, + "username", + "API Key", + server.URL, + ) + + if err != nil { + t.Fatal(err) + } + + c := newSpaceBuilder(confluenceClient) + + t.Run("should list spaces", func(t *testing.T) { + resources := make([]*v2.Resource, 0) + pToken := pagination.Token{} + for { + nextResources, nextToken, listAnnotations, err := c.List(ctx, nil, &pToken) + resources = append(resources, nextResources...) + + require.Nil(t, err) + test.AssertNoRatelimitAnnotations(t, listAnnotations) + if nextToken == "" { + break + } + pToken.Token = nextToken + } + + require.Nil(t, err) + require.Len(t, resources, 2) + require.NotEmpty(t, resources[0].Id) + }) + + t.Run("should list grants for a space", func(t *testing.T) { + confluenceSpace := client.ConfluenceSpace{ + Id: "678", + } + space, _ := spaceResource(ctx, &confluenceSpace) + + grants, nextToken, grantsAnnotations, err := c.Grants(ctx, space, &pagination.Token{}) + require.Nil(t, err) + test.AssertNoRatelimitAnnotations(t, grantsAnnotations) + require.Equal(t, "", nextToken) + require.Len(t, grants, 25) + }) +} From 3e2c4481413f58449604ce61c7993560b099ee05 Mon Sep 17 00:00:00 2001 From: Marcos Gaeta Date: Thu, 25 Jul 2024 09:40:50 -0700 Subject: [PATCH 2/6] return ratelimit data on successful space permission retrieval --- pkg/connector/client/confluence.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/connector/client/confluence.go b/pkg/connector/client/confluence.go index a56c3d9b..a71c2962 100644 --- a/pkg/connector/client/confluence.go +++ b/pkg/connector/client/confluence.go @@ -574,7 +574,7 @@ func (c *ConfluenceClient) findSpacePermission( permission.Principal.Type == principalType && permission.Operation.Key == key && permission.Operation.TargetType == target { - return &permission, nil, nil + return &permission, ratelimitData, nil } } cursor = extractPaginationCursor(response.Links) From d3e7dc790e2c27c60efe434737d895446180081b Mon Sep 17 00:00:00 2001 From: Marcos Gaeta Date: Thu, 25 Jul 2024 10:23:42 -0700 Subject: [PATCH 3/6] Add space permission info to README --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index da7c2c1e..0caab1af 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,64 @@ baton resources # Data Model `baton-confluence` will pull down information about the following Confluence resources: +- Spaces & Space Permissions - Groups - Users +## Space Permissions +Every Confluence space has its own set of permissions which determine what +people can do in the space. + +These permissions are Entitlements for Spaces and are represented as a pair of +"operation" and "target". + +Valid targets include: +- `application` +- `attachment` +- `blogpost` +- `comment` +- `database` +- `embed` +- `page` +- `space` +- `userProfile` +- `whiteboard` + +Valid operations include: +- `administer` +- `archive` +- `copy` +- `create` +- `create_space` +- `delete` +- `export` +- `move` +- `purge` +- `purge_version` +- `read` +- `restore` +- `restrict_content` +- `update` + +Not all operation-target pairs are valid, but here are some examples of valid ones: +- `administer-space` +- `create-attachment` +- `create-blogpost` +- `create-comment` +- `create-page` +- `delete-space` +- `export-space` +- `read-space` +- `update-space` + +See [Space Permissions Overview documentation page](https://confluence.atlassian.com/doc/space-permissions-overview-139521.html). + # Contributing, Support and Issues -We started Baton because we were tired of taking screenshots and manually building spreadsheets. We welcome contributions, and ideas, no matter how small -- our goal is to make identity and permissions sprawl less painful for everyone. If you have questions, problems, or ideas: Please open a Github Issue! +We started Baton because we were tired of taking screenshots and manually +building spreadsheets. We welcome contributions, and ideas, no matter how small +-- our goal is to make identity and permissions sprawl less painful for +everyone. If you have questions, problems, or ideas: Please open a GitHub Issue! See [CONTRIBUTING.md](https://github.com/ConductorOne/baton/blob/main/CONTRIBUTING.md) for more details. @@ -73,5 +125,4 @@ Flags: -v, --version version for baton-confluence Use "baton-confluence [command] --help" for more information about a command. - ``` From ca164f569cbce39661e7dc87306f013bc826de7c Mon Sep 17 00:00:00 2001 From: Marcos Gaeta Date: Thu, 25 Jul 2024 11:27:59 -0700 Subject: [PATCH 4/6] smarter url parsing --- pkg/connector/client/confluence.go | 203 ++++------------------------- pkg/connector/client/path.go | 4 +- pkg/connector/client/ratelimit.go | 45 +++++++ pkg/connector/client/request.go | 86 ++++++++++++ 4 files changed, 159 insertions(+), 179 deletions(-) create mode 100644 pkg/connector/client/ratelimit.go create mode 100644 pkg/connector/client/request.go diff --git a/pkg/connector/client/confluence.go b/pkg/connector/client/confluence.go index a71c2962..95de9f06 100644 --- a/pkg/connector/client/confluence.go +++ b/pkg/connector/client/confluence.go @@ -5,20 +5,14 @@ import ( "encoding/json" "errors" "fmt" - "io" - "net/http" "net/url" - "slices" "strconv" "strings" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" - "github.com/conductorone/baton-sdk/pkg/helpers" "github.com/conductorone/baton-sdk/pkg/uhttp" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) const ( @@ -79,7 +73,7 @@ func NewConfluenceClient(ctx context.Context, user, apiKey, domain string) (*Con } func (c *ConfluenceClient) Verify(ctx context.Context) error { - currentUserUrl, err := c.genURLNonPaginated(CurrentUserUrlPath) + currentUserUrl, err := c.parse(CurrentUserUrlPath) if err != nil { return err } @@ -112,7 +106,10 @@ func (c *ConfluenceClient) GetGroups( *v2.RateLimitDescription, error, ) { - groupsUrl, err := c.genURL(pageToken, pageSize, GroupsListUrlPath) + groupsUrl, err := c.parse( + GroupsListUrlPath, + withLimitAndOffset(pageToken, pageSize), + ) if err != nil { return nil, "", nil, err } @@ -178,11 +175,9 @@ func (c *ConfluenceClient) AddUserToGroup( accountID string, groupId string, ) (*v2.RateLimitDescription, error) { - getUsersUrl, err := c.genURLNonPaginated( - fmt.Sprintf( - addUsersToGroupUrlPath, - groupId, - ), + getUsersUrl, err := c.parse( + groupBaseUrlPath, + withQueryParameters(map[string]interface{}{"groupId": groupId}), ) if err != nil { return nil, err @@ -210,12 +205,12 @@ func (c *ConfluenceClient) RemoveUserFromGroup( accountID string, groupId string, ) (*v2.RateLimitDescription, error) { - getUsersUrl, err := c.genURLNonPaginated( - fmt.Sprintf( - removeUsersFromGroupUrlPath, - groupId, - accountID, - ), + getUsersUrl, err := c.parse( + groupBaseUrlPath, + withQueryParameters(map[string]interface{}{ + "groupId": groupId, + "accountId": accountID, + }), ) if err != nil { return nil, err @@ -228,112 +223,6 @@ func (c *ConfluenceClient) RemoveUserFromGroup( return ratelimitData, nil } -func (c *ConfluenceClient) get( - ctx context.Context, - getUrl *url.URL, - target interface{}, -) (*v2.RateLimitDescription, error) { - return c.makeRequest(ctx, getUrl, target, http.MethodGet, nil) -} - -func (c *ConfluenceClient) post( - ctx context.Context, - postUrl *url.URL, - target interface{}, - requestBody io.Reader, -) (*v2.RateLimitDescription, error) { - return c.makeRequest(ctx, postUrl, target, http.MethodPost, requestBody) -} - -func (c *ConfluenceClient) delete( - ctx context.Context, - deleteUrl *url.URL, - target interface{}, -) (*v2.RateLimitDescription, error) { - return c.makeRequest(ctx, deleteUrl, target, http.MethodDelete, nil) -} - -func (c *ConfluenceClient) makeRequest( - ctx context.Context, - url *url.URL, - target interface{}, - method string, - requestBody io.Reader, -) (*v2.RateLimitDescription, error) { - req, err := http.NewRequestWithContext(ctx, method, url.String(), requestBody) - if err != nil { - return nil, err - } - - req.SetBasicAuth(c.user, c.apiKey) - - ratelimitData := v2.RateLimitDescription{} - - response, err := c.wrapper.Do( - req, - WithConfluenceRatelimitData(&ratelimitData), - uhttp.WithJSONResponse(target), - ) - if err == nil { - return &ratelimitData, nil - } - if response == nil { - return nil, err - } - defer response.Body.Close() - - // If we get ratelimit data back (e.g. the "Retry-After" header) or a - // "ratelimit-like" status code, then return a recoverable gRPC code. - if isRatelimited(ratelimitData.Status, response.StatusCode) { - return &ratelimitData, status.Error(codes.Unavailable, response.Status) - } - - // If it's some other error, it is unrecoverable. - responseBody, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - - return nil, &RequestError{ - URL: url, - Status: response.StatusCode, - Body: string(responseBody), - } -} - -// genURLNonPaginated adds the given URL path to the API base URL. -func (c *ConfluenceClient) genURLNonPaginated(path string) (*url.URL, error) { - parsed, err := url.Parse(path) - if err != nil { - return nil, fmt.Errorf("failed to parse request path '%s': %w", path, err) - } - parsedUrl := c.apiBase.ResolveReference(parsed) - return parsedUrl, nil -} - -// genURL adds `start` and `limit` query parameters to a URL. This pagination -// parameter is only used by the v1 REST API. -func (c *ConfluenceClient) genURL(pageToken string, pageSize int, path string) (*url.URL, error) { - parsed, err := url.Parse(path) - if err != nil { - return nil, fmt.Errorf("failed to parse request path '%s': %w", path, err) - } - - parsedUrl := c.apiBase.ResolveReference(parsed) - - maximum := pageSize - if maximum == 0 || maximum > maxResults { - maximum = maxResults - } - - query := parsedUrl.Query() - query.Set("start", pageToken) - query.Set("limit", strconv.Itoa(maximum)) - parsedUrl.RawQuery = query.Encode() - - return parsedUrl, nil -} - func incToken(pageToken string, count int) string { token := strToInt(pageToken) @@ -364,10 +253,9 @@ func (c *ConfluenceClient) GetSpaces( *v2.RateLimitDescription, error, ) { - spacesListUrl, err := c.genURLWithPaginationCursor( + spacesListUrl, err := c.parse( SpacesListUrlPath, - pageSize, - paginationCursor, + withPaginationCursor(pageSize, paginationCursor), ) if err != nil { return nil, "", nil, err @@ -399,10 +287,10 @@ func (c *ConfluenceClient) ConfluenceSpaceOperations( logger := ctxzap.Extract(ctx) logger.Debug("fetching space", zap.String("spaceId", spaceId)) - spaceUrl, err := c.genURLWithPaginationCursor( - fmt.Sprintf(spacesGetUrlPath+"?include-operations=1", spaceId), - pageSize, - cursor, + spaceUrl, err := c.parse( + fmt.Sprintf(spacesGetUrlPath, spaceId), + withQueryParameters(map[string]interface{}{"include-operations": true}), + withPaginationCursor(pageSize, cursor), ) if err != nil { @@ -437,10 +325,9 @@ func (c *ConfluenceClient) GetSpacePermissions( *v2.RateLimitDescription, error, ) { - spacePermissionsListUrl, err := c.genURLWithPaginationCursor( + spacePermissionsListUrl, err := c.parse( fmt.Sprintf(SpacePermissionsListUrlPath, spaceId), - pageSize, - pageToken, + withPaginationCursor(pageSize, pageToken), ) if err != nil { return nil, "", nil, err @@ -487,7 +374,7 @@ func (c *ConfluenceClient) AddSpacePermission( *v2.RateLimitDescription, error, ) { - spacePermissionsListUrl, err := c.genURLNonPaginated( + spacePermissionsListUrl, err := c.parse( fmt.Sprintf(spacePermissionsCreateUrlPath, spaceName), ) if err != nil { @@ -548,13 +435,12 @@ func (c *ConfluenceClient) findSpacePermission( // We need to list _all_ permissions in order to figure out the permission's ID. cursor := "" for { - listPermissionsUrl, err := c.genURLWithPaginationCursor( + listPermissionsUrl, err := c.parse( fmt.Sprintf( SpacePermissionsListUrlPath, spaceId, ), - maxResults, - cursor, + withPaginationCursor(maxResults, cursor), ) if err != nil { return nil, nil, err @@ -596,7 +482,7 @@ func (c *ConfluenceClient) findSpace( *v2.RateLimitDescription, error, ) { - getSpaceUrl, err := c.genURLNonPaginated( + getSpaceUrl, err := c.parse( fmt.Sprintf( spacesGetUrlPath, spaceId, @@ -647,7 +533,7 @@ func (c *ConfluenceClient) RemoveSpacePermission( return ratelimitData, err } - deletePermissionUrl, err := c.genURLNonPaginated( + deletePermissionUrl, err := c.parse( fmt.Sprintf( spacePermissionsUpdateUrlPath, space.Key, @@ -681,41 +567,6 @@ func extractPaginationCursor(links ConfluenceLink) string { return parsedUrl.Query().Get("cursor") } -// WithConfluenceRatelimitData Per the docs: transient 5XX errors should be -// treated as 429/too-many-requests if they have a retry header. 503 errors were -// the only ones explicitly called out, but I guess it's possible for others too -// https://developer.atlassian.com/cloud/confluence/rate-limiting/ -func WithConfluenceRatelimitData(resource *v2.RateLimitDescription) uhttp.DoOption { - return func(response *uhttp.WrapperResponse) error { - rateLimitData, err := helpers.ExtractRateLimitData(response.StatusCode, &response.Header) - if err != nil { - return err - } - resource = rateLimitData - return nil - } -} - -func isRatelimited( - ratelimitStatus v2.RateLimitDescription_Status, - statusCode int, -) bool { - return slices.Contains( - []v2.RateLimitDescription_Status{ - v2.RateLimitDescription_STATUS_OVERLIMIT, - v2.RateLimitDescription_STATUS_ERROR, - }, - ratelimitStatus, - ) || slices.Contains( - []int{ - http.StatusTooManyRequests, - http.StatusGatewayTimeout, - http.StatusServiceUnavailable, - }, - statusCode, - ) -} - // GetUsersFromSearch There are no official, documented ways to get lists of // users in Confluence. One way to get users is to issue a CQL search query with // no conditions. The documentation mentions that queries return "up to 10k" diff --git a/pkg/connector/client/path.go b/pkg/connector/client/path.go index f8ef2f74..e2839cc9 100644 --- a/pkg/connector/client/path.go +++ b/pkg/connector/client/path.go @@ -11,14 +11,12 @@ const ( GroupsListUrlPath = "/wiki/rest/api/group" getUsersByGroupIdUrlPath = "/wiki/rest/api/group/%s/membersByGroupId" groupBaseUrlPath = "/wiki/rest/api/group/userByGroupId" + SearchUrlPath = "/wiki/rest/api/search/user" spacePermissionsCreateUrlPath = "/wiki/rest/api/space/%s/permissions" spacePermissionsUpdateUrlPath = "/wiki/rest/api/space/%s/permissions/%s" SpacesListUrlPath = "/wiki/api/v2/spaces" spacesGetUrlPath = "/wiki/api/v2/spaces/%s" SpacePermissionsListUrlPath = "/wiki/api/v2/spaces/%s/permissions" - SearchUrlPath = "/wiki/rest/api/search/user" - addUsersToGroupUrlPath = "/wiki/rest/api/group/userByGroupId?groupId=%s" - removeUsersFromGroupUrlPath = "/wiki/rest/api/group/userByGroupId?groupId=%s&accountId=%s" ) type Option = func(*url.URL) (*url.URL, error) diff --git a/pkg/connector/client/ratelimit.go b/pkg/connector/client/ratelimit.go new file mode 100644 index 00000000..760c1c3a --- /dev/null +++ b/pkg/connector/client/ratelimit.go @@ -0,0 +1,45 @@ +package client + +import ( + "net/http" + "slices" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/helpers" + "github.com/conductorone/baton-sdk/pkg/uhttp" +) + +// WithConfluenceRatelimitData Per the docs: transient 5XX errors should be +// treated as 429/too-many-requests if they have a retry header. 503 errors were +// the only ones explicitly called out, but I guess it's possible for others too +// https://developer.atlassian.com/cloud/confluence/rate-limiting/ +func WithConfluenceRatelimitData(resource *v2.RateLimitDescription) uhttp.DoOption { + return func(response *uhttp.WrapperResponse) error { + rateLimitData, err := helpers.ExtractRateLimitData(response.StatusCode, &response.Header) + if err != nil { + return err + } + resource = rateLimitData + return nil + } +} + +func isRatelimited( + ratelimitStatus v2.RateLimitDescription_Status, + statusCode int, +) bool { + return slices.Contains( + []v2.RateLimitDescription_Status{ + v2.RateLimitDescription_STATUS_OVERLIMIT, + v2.RateLimitDescription_STATUS_ERROR, + }, + ratelimitStatus, + ) || slices.Contains( + []int{ + http.StatusTooManyRequests, + http.StatusGatewayTimeout, + http.StatusServiceUnavailable, + }, + statusCode, + ) +} diff --git a/pkg/connector/client/request.go b/pkg/connector/client/request.go new file mode 100644 index 00000000..01ec493a --- /dev/null +++ b/pkg/connector/client/request.go @@ -0,0 +1,86 @@ +package client + +import ( + "context" + "io" + "net/http" + "net/url" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/uhttp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (c *ConfluenceClient) get( + ctx context.Context, + getUrl *url.URL, + target interface{}, +) (*v2.RateLimitDescription, error) { + return c.makeRequest(ctx, getUrl, target, http.MethodGet, nil) +} + +func (c *ConfluenceClient) post( + ctx context.Context, + postUrl *url.URL, + target interface{}, + requestBody io.Reader, +) (*v2.RateLimitDescription, error) { + return c.makeRequest(ctx, postUrl, target, http.MethodPost, requestBody) +} + +func (c *ConfluenceClient) delete( + ctx context.Context, + deleteUrl *url.URL, + target interface{}, +) (*v2.RateLimitDescription, error) { + return c.makeRequest(ctx, deleteUrl, target, http.MethodDelete, nil) +} + +func (c *ConfluenceClient) makeRequest( + ctx context.Context, + url *url.URL, + target interface{}, + method string, + requestBody io.Reader, +) (*v2.RateLimitDescription, error) { + req, err := http.NewRequestWithContext(ctx, method, url.String(), requestBody) + if err != nil { + return nil, err + } + + req.SetBasicAuth(c.user, c.apiKey) + + ratelimitData := v2.RateLimitDescription{} + + response, err := c.wrapper.Do( + req, + WithConfluenceRatelimitData(&ratelimitData), + uhttp.WithJSONResponse(target), + ) + if err == nil { + return &ratelimitData, nil + } + if response == nil { + return nil, err + } + defer response.Body.Close() + + // If we get ratelimit data back (e.g. the "Retry-After" header) or a + // "ratelimit-like" status code, then return a recoverable gRPC code. + if isRatelimited(ratelimitData.Status, response.StatusCode) { + return &ratelimitData, status.Error(codes.Unavailable, response.Status) + } + + // If it's some other error, it is unrecoverable. + responseBody, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + return nil, &RequestError{ + URL: url, + Status: response.StatusCode, + Body: string(responseBody), + } +} From 09bac3e6f09b628cac4c24d1feadab551e283993 Mon Sep 17 00:00:00 2001 From: Marcos Gaeta Date: Fri, 2 Aug 2024 16:25:42 -0700 Subject: [PATCH 5/6] lint --- pkg/connector/client/path.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/connector/client/path.go b/pkg/connector/client/path.go index e2839cc9..f9d3d186 100644 --- a/pkg/connector/client/path.go +++ b/pkg/connector/client/path.go @@ -56,9 +56,8 @@ func withLimitAndOffset(pageToken string, pageSize int) Option { }) } -// WithPaginationCursor uses Confluence Cloud's REST API v2 pagination scheme. -func WithPaginationCursor( - pageSize int, +// withPaginationCursor uses Confluence Cloud's REST API v2 pagination scheme. +func withPaginationCursor(pageSize int, paginationCursor string, ) Option { parameters := map[string]interface{}{ From 004c88d7d09ea997ebf185149d2bbb7bb4d44625 Mon Sep 17 00:00:00 2001 From: Marcos Gaeta Date: Tue, 6 Aug 2024 11:01:51 -0700 Subject: [PATCH 6/6] set the attributes on the struct one by one --- pkg/connector/client/ratelimit.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/connector/client/ratelimit.go b/pkg/connector/client/ratelimit.go index 760c1c3a..893d4223 100644 --- a/pkg/connector/client/ratelimit.go +++ b/pkg/connector/client/ratelimit.go @@ -19,7 +19,10 @@ func WithConfluenceRatelimitData(resource *v2.RateLimitDescription) uhttp.DoOpti if err != nil { return err } - resource = rateLimitData + resource.Limit = rateLimitData.Limit + resource.Remaining = rateLimitData.Remaining + resource.ResetAt = rateLimitData.ResetAt + resource.Status = rateLimitData.Status return nil } }