From 508766985ae8a0f1be5acab23ff6ecc17331f320 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 11 Jun 2024 23:18:51 +0000 Subject: [PATCH] Merge remote-tracking branch 'harness0/master' into abhinav/add-exporter (#7) --- cmd/cmd.go | 3 + cmd/gitimporter/git.go | 159 ++++++++++++++++++ cmd/gitimporter/importer.go | 22 +++ cmd/stash/git.go | 6 +- go.mod | 1 + go.sum | 2 + internal/{common => gitexporter}/common.go | 2 +- internal/{common => gitexporter}/exporter.go | 46 +++-- internal/{common => gitexporter}/interface.go | 2 +- internal/gitimporter/check.go | 73 ++++++++ internal/gitimporter/importer.go | 50 ++++++ internal/gitimporter/upload.go | 38 +++++ internal/gitimporter/users.go | 20 +++ internal/harness/client.go | 19 ++- internal/harness/client_impl.go | 145 ++++++++++++++++ internal/migrate/stash/stash.go | 3 +- internal/types/types.go | 1 + types/git_exporter.go | 30 ++++ types/git_importer.go | 61 +++++++ 19 files changed, 659 insertions(+), 24 deletions(-) create mode 100644 cmd/gitimporter/git.go create mode 100644 cmd/gitimporter/importer.go rename internal/{common => gitexporter}/common.go (97%) rename internal/{common => gitexporter}/exporter.go (72%) rename internal/{common => gitexporter}/interface.go (98%) create mode 100644 internal/gitimporter/check.go create mode 100644 internal/gitimporter/importer.go create mode 100644 internal/gitimporter/upload.go create mode 100644 internal/gitimporter/users.go create mode 100644 types/git_exporter.go create mode 100644 types/git_importer.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 37e6af9..edbc467 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -22,6 +22,7 @@ import ( "github.com/harness/harness-migrate/cmd/cloudbuild" "github.com/harness/harness-migrate/cmd/drone" "github.com/harness/harness-migrate/cmd/github" + "github.com/harness/harness-migrate/cmd/gitimporter" "github.com/harness/harness-migrate/cmd/gitlab" "github.com/harness/harness-migrate/cmd/stash" "github.com/harness/harness-migrate/cmd/terraform" @@ -54,6 +55,8 @@ func Command() { terraform.Register(app) stash.Register(app) + gitimporter.Register(app) + app.Version(version) kingpin.MustParse(app.Parse(os.Args[1:])) } diff --git a/cmd/gitimporter/git.go b/cmd/gitimporter/git.go new file mode 100644 index 0000000..4637846 --- /dev/null +++ b/cmd/gitimporter/git.go @@ -0,0 +1,159 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitimporter + +import ( + "bufio" + "context" + "fmt" + "os" + "strings" + + "github.com/harness/harness-migrate/cmd/util" + "github.com/harness/harness-migrate/internal/gitimporter" + "github.com/harness/harness-migrate/internal/tracer" + "github.com/harness/harness-migrate/types" + + "github.com/alecthomas/kingpin/v2" + "github.com/google/uuid" + "golang.org/x/exp/slog" +) + +type gitImport struct { + debug bool + trace bool + + harnessEndpoint string + harnessToken string + harnessSpace string + //harnessRepo string + + filePath string +} + +type UserInvite bool + +func (c *gitImport) run(*kingpin.ParseContext) error { + // create the logger + log := util.CreateLogger(c.debug) + + // attach the logger to the context + ctx := context.Background() + ctx = slog.NewContext(ctx, log) + + tracer_ := tracer.New() + defer tracer_.Close() + + importUuid := uuid.New().String() + importer := gitimporter.NewImporter(c.harnessSpace, c.harnessToken, c.filePath, importUuid, tracer_) + + tracer_.Log("starting operation with id: %s", importUuid) + + repositoriesImportOutput, err := importer.UploadZip() + + if err != nil { + tracer_.LogError("encountered error uploading zip: %s", err) + return err + } + + if err := checkAndPerformUserInvite(repositoriesImportOutput, tracer_, importer); err != nil { + return err + } + + if err := importer.IsComplete(); err != nil { + return err + } + return nil +} + +func checkAndPerformUserInvite(repositoriesImportOutput *types.RepositoriesImportOutput, tracer_ tracer.Tracer, importer *gitimporter.Importer) error { + if repositoriesImportOutput != nil && len(repositoriesImportOutput.Users.NotPresent) != 0 { + tracer_.Log("Found users which are not in harness and are present in import data: ") + tracer_.Log("%v", repositoriesImportOutput.Users.NotPresent) + userFollowUp, err := doUserFollowUp() + if err != nil { + return err + } + if userFollowUp { + err = importer.InviteUsers(repositoriesImportOutput.Users.NotPresent) + if err != nil { + return err + } + } + } + return nil +} + +func doUserFollowUp() (UserInvite, error) { + reader := bufio.NewReader(os.Stdin) + + fmt.Println("Please select one of the following options:") + fmt.Println("1. Map missing user to yourself") + fmt.Println("2. Invite missing users (needs admin permission for space)") + fmt.Print("Enter your choice (1 or 2): ") + + choice, _ := reader.ReadString('\n') + choice = strings.TrimSpace(choice) + + switch choice { + case "1": + fmt.Println("You selected Option 1") + return false, nil + case "2": + fmt.Println("You selected Option 2") + return true, nil + default: + fmt.Println("Invalid choice. Please enter 1 or 2.") + return false, fmt.Errorf("invalid option selected for user invite") + } +} + +func registerGitImporter(app *kingpin.CmdClause) { + c := new(gitImport) + + cmd := app.Command("git-import", "import git data into harness/gitness"). + Hidden(). + Action(c.run) + + cmd.Arg("filePath", "location of the zip file"). + Required(). + Envar("HARNESS_FILEPATH"). + StringVar(&c.filePath) + + cmd.Flag("harnessEndpoint", "url of harness code host"). + Default("https://app.harness.io/"). + Envar("HARNESS_HOST"). + StringVar(&c.harnessEndpoint) + + cmd.Flag("token", "harness api token"). + Required(). + Envar("HARNESS_TOKEN"). + StringVar(&c.harnessToken) + + cmd.Flag("space", "harness path where import should take place. Example: account/org/project"). + Required(). + Envar("HARNESS_TOKEN"). + StringVar(&c.harnessSpace) + + // cmd.Flag("repo", "Required in case of single repo import which already exists."). + // Envar("HARNESS_REPO"). + // StringVar(&c.harnessRepo) + + cmd.Flag("debug", "enable debug logging"). + BoolVar(&c.debug) + + cmd.Flag("trace", "enable trace logging"). + BoolVar(&c.trace) +} diff --git a/cmd/gitimporter/importer.go b/cmd/gitimporter/importer.go new file mode 100644 index 0000000..9b089c6 --- /dev/null +++ b/cmd/gitimporter/importer.go @@ -0,0 +1,22 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitimporter + +import "github.com/alecthomas/kingpin/v2" + +func Register(app *kingpin.Application) { + cmd := app.Command("git-import", "migrate data into harness from exported zip") + registerGitImporter(cmd) +} diff --git a/cmd/stash/git.go b/cmd/stash/git.go index b0b2af3..d2a7801 100644 --- a/cmd/stash/git.go +++ b/cmd/stash/git.go @@ -21,7 +21,7 @@ import ( "github.com/harness/harness-migrate/cmd/util" "github.com/harness/harness-migrate/internal/checkpoint" - "github.com/harness/harness-migrate/internal/common" + "github.com/harness/harness-migrate/internal/gitexporter" "github.com/harness/harness-migrate/internal/migrate/stash" "github.com/harness/harness-migrate/internal/tracer" @@ -88,7 +88,7 @@ func (c *exportCommand) run(*kingpin.ParseContext) error { // extract the data e := stash.New(client, c.stashOrg, checkpointManager, tracer_) - exporter := common.NewExporter(e, c.file) + exporter := gitexporter.NewExporter(e, c.file) exporter.Export(ctx) return nil } @@ -97,7 +97,7 @@ func (c *exportCommand) run(*kingpin.ParseContext) error { func registerGit(app *kingpin.CmdClause) { c := new(exportCommand) - cmd := app.Command("git", "export stash git data"). + cmd := app.Command("git-export", "export stash git data"). Hidden(). Action(c.run) diff --git a/go.mod b/go.mod index fee6b5e..c6b2ee1 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect diff --git a/go.sum b/go.sum index 0a87d2e..f1c3c9a 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gotidy/ptr v1.4.0 h1:7++suUs+HNHMnyz6/AW3SE+4EnBhupPSQTSI7QNijVc= github.com/gotidy/ptr v1.4.0/go.mod h1:MjRBG6/IETiiZGWI8LrRtISXEji+8b/jigmj2q0mEyM= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= diff --git a/internal/common/common.go b/internal/gitexporter/common.go similarity index 97% rename from internal/common/common.go rename to internal/gitexporter/common.go index 8dacb17..a86987e 100644 --- a/internal/common/common.go +++ b/internal/gitexporter/common.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package gitexporter import ( "github.com/harness/harness-migrate/internal/types" diff --git a/internal/common/exporter.go b/internal/gitexporter/exporter.go similarity index 72% rename from internal/common/exporter.go rename to internal/gitexporter/exporter.go index bda5eeb..c0f84ad 100644 --- a/internal/common/exporter.go +++ b/internal/gitexporter/exporter.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package gitexporter import ( "context" @@ -23,13 +23,16 @@ import ( "path/filepath" "github.com/harness/harness-migrate/internal/codeerror" + "github.com/harness/harness-migrate/internal/common" "github.com/harness/harness-migrate/internal/types" "github.com/harness/harness-migrate/internal/util" + + externalTypes "github.com/harness/harness-migrate/types" ) const ( maxChunkSize = 25 * 1024 * 1024 // 25 MB - infoFileName = "info.json" + prFileName = "pr%d.json" ) type Exporter struct { @@ -46,19 +49,19 @@ func (e *Exporter) Export(ctx context.Context) { path := filepath.Join(".", e.zipLocation) err := util.CreateFolder(path) if err != nil { - panic(fmt.Sprintf(PanicCannotCreateFolder, err)) + panic(fmt.Sprintf(common.PanicCannotCreateFolder, err)) } data, _ := e.getData(ctx) for _, repo := range data { - err = e.writeJsonForRepo(repo) + err = e.writeJsonForRepo(mapRepoData(repo)) if err != nil { - panic(fmt.Sprintf(PanicWritingFileData, err)) + panic(fmt.Sprintf(common.PanicWritingFileData, err)) } } } // Calculate the size of the struct in bytes -func calculateSize(s *types.PullRequestData) int { +func calculateSize(s *externalTypes.PullRequestData) int { data, err := json.Marshal(s) // will never happen if err != nil { @@ -68,16 +71,16 @@ func calculateSize(s *types.PullRequestData) int { } // Split the array into smaller chunks if the size exceeds the maxChunkSize -func splitArray(arr []*types.PullRequestData) [][]*types.PullRequestData { - var chunks [][]*types.PullRequestData - var currentChunk []*types.PullRequestData +func splitArray(arr []*externalTypes.PullRequestData) [][]*externalTypes.PullRequestData { + var chunks [][]*externalTypes.PullRequestData + var currentChunk []*externalTypes.PullRequestData currentSize := 0 for _, item := range arr { itemSize := calculateSize(item) if currentSize+itemSize > maxChunkSize { chunks = append(chunks, currentChunk) - currentChunk = []*types.PullRequestData{} + currentChunk = []*externalTypes.PullRequestData{} currentSize = 0 } currentChunk = append(currentChunk, item) @@ -89,15 +92,15 @@ func splitArray(arr []*types.PullRequestData) [][]*types.PullRequestData { return chunks } -func (e *Exporter) writeJsonForRepo(repo *types.RepoData) error { +func (e *Exporter) writeJsonForRepo(repo *externalTypes.RepositoryData) error { repoJson, _ := util.GetJson(repo.Repository) - pathRepo := filepath.Join(".", e.zipLocation, repo.Repository.RepoSlug) + pathRepo := filepath.Join(".", e.zipLocation, repo.Repository.Slug) err := util.CreateFolder(pathRepo) if err != nil { return fmt.Errorf("cannot create folder") } - err = util.WriteFile(filepath.Join(pathRepo, infoFileName), repoJson) + err = util.WriteFile(filepath.Join(pathRepo, externalTypes.InfoFileName), repoJson) if err != nil { return err } @@ -107,7 +110,7 @@ func (e *Exporter) writeJsonForRepo(repo *types.RepoData) error { } prDataInSize := splitArray(repo.PullRequestData) - pathPR := filepath.Join(pathRepo, "pr") + pathPR := filepath.Join(pathRepo, externalTypes.PRFolderName) err = util.CreateFolder(pathPR) if err != nil { return err @@ -119,7 +122,7 @@ func (e *Exporter) writeJsonForRepo(repo *types.RepoData) error { // todo: fix this log.Printf("cannot serialize into json: %v", err) } - prFilePath := fmt.Sprintf("pr%d.json", i) + prFilePath := fmt.Sprintf(prFileName, i) err = util.WriteFile(filepath.Join(pathPR, prFilePath), prJson) if err != nil { @@ -174,3 +177,16 @@ func mapPrData(pr types.PRResponse, comments []types.PRComments) *types.PullRequ Comments: comments, } } + +func mapRepoData(repoData *types.RepoData) *externalTypes.RepositoryData { + d := new(externalTypes.RepositoryData) + d.Repository.Slug = repoData.Repository.RepoSlug + d.Repository.Repository = repoData.Repository.Repository + + d.PullRequestData = make([]*externalTypes.PullRequestData, len(repoData.PullRequestData)) + for i, prData := range repoData.PullRequestData { + d.PullRequestData[i].PullRequest.PullRequest = prData.PullRequest.PullRequest + // todo: map comment data + } + return d +} diff --git a/internal/common/interface.go b/internal/gitexporter/interface.go similarity index 98% rename from internal/common/interface.go rename to internal/gitexporter/interface.go index e9afb32..32fcc11 100644 --- a/internal/common/interface.go +++ b/internal/gitexporter/interface.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package gitexporter import ( "context" diff --git a/internal/gitimporter/check.go b/internal/gitimporter/check.go new file mode 100644 index 0000000..56f88d8 --- /dev/null +++ b/internal/gitimporter/check.go @@ -0,0 +1,73 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitimporter + +import ( + "context" + "fmt" + "time" + + "github.com/harness/harness-migrate/types" +) + +const ( + pollInterval = 5 * time.Second + pollTimeout = 5 * time.Minute +) + +func (c *Importer) IsComplete() error { + c.Tracer.Start("Performing import on harness code") + start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), pollTimeout) + defer cancel() + + // todo: handle context cancelled due to timeout. + err := c.pollOperationStatus(ctx) + if err != nil { + c.Tracer.Stop("Import error: %s", err) + return err + } + c.Tracer.Stop("Import complete in %s seconds", time.Since(start).Seconds()) + + return nil +} + +func (c *Importer) pollOperationStatus(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + complete, err := c.checkOperationStatus() + if err != nil { + return err + } + if complete { + fmt.Println("Operation is complete") + return nil + } + fmt.Println("Operation is not complete, waiting to poll again...") + time.Sleep(pollInterval) + } + + return fmt.Errorf("operation did not complete within the expected time") +} + +func (c *Importer) checkOperationStatus() (bool, error) { + checkImport, err := c.Harness.HarnessCodeCheckImport(c.HarnessSpace, c.RequestId) + if err != nil { + return false, fmt.Errorf("error checking status: %w", err) + } + return checkImport.Status == types.RepoImportStatusComplete, nil +} diff --git a/internal/gitimporter/importer.go b/internal/gitimporter/importer.go new file mode 100644 index 0000000..0fd2626 --- /dev/null +++ b/internal/gitimporter/importer.go @@ -0,0 +1,50 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitimporter + +import ( + "strings" + + "github.com/harness/harness-migrate/internal/harness" + "github.com/harness/harness-migrate/internal/tracer" +) + +// Importer imports data from gitlab to Harness. +type Importer struct { + Harness harness.Client + + HarnessSpace string + HarnessToken string + + ZipFileLocation string + + Tracer tracer.Tracer + + RequestId string +} + +func NewImporter(space, token, location, requestId string, tracer tracer.Tracer) *Importer { + spaceSplit := strings.Split(space, "/") + client := harness.New(spaceSplit[0], token) + + return &Importer{ + Harness: client, + HarnessSpace: space, + HarnessToken: token, + ZipFileLocation: location, + Tracer: tracer, + RequestId: requestId, + } +} diff --git a/internal/gitimporter/upload.go b/internal/gitimporter/upload.go new file mode 100644 index 0000000..65ff911 --- /dev/null +++ b/internal/gitimporter/upload.go @@ -0,0 +1,38 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitimporter + +import ( + "fmt" + "time" + + "github.com/harness/harness-migrate/types" +) + +func (c *Importer) UploadZip() (*types.RepositoriesImportOutput, error) { + c.Tracer.Start("starting uploading zip file") + in := &types.RepositoriesImportInput{ + RequestId: c.RequestId, + } + + start := time.Now() + out, err := c.Harness.UploadHarnessCodeZip(c.HarnessSpace, c.ZipFileLocation, c.RequestId, in) + if err != nil { + c.Tracer.Stop("error uploading zip") + return nil, fmt.Errorf("error uploading zip: %w", err) + } + c.Tracer.Stop("zip upload complete in %d seconds", time.Since(start).Seconds()) + return out, nil +} diff --git a/internal/gitimporter/users.go b/internal/gitimporter/users.go new file mode 100644 index 0000000..ecb67cb --- /dev/null +++ b/internal/gitimporter/users.go @@ -0,0 +1,20 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitimporter + +func (c *Importer) InviteUsers(users []string) error { + // todo: implement + return nil +} diff --git a/internal/harness/client.go b/internal/harness/client.go index 9a91e7d..50e6055 100644 --- a/internal/harness/client.go +++ b/internal/harness/client.go @@ -15,7 +15,11 @@ // Package harness provides a Harness http client. package harness -import "time" +import ( + "time" + + "github.com/harness/harness-migrate/types" +) // Client is used to communicate with the Harness server. type Client interface { @@ -55,13 +59,13 @@ type Client interface { // CreateSecret creates a secret. CreateSecret(secret *Secret) error - // CreateSecret creates an organization secret. + // CreateSecretOrg creates an organization secret. CreateSecretOrg(secret *Secret) error // CreateConnector creates a connector. CreateConnector(connector *Connector) error - // CreateConnector creates an organization connector. + // CreateConnectorOrg creates an organization connector. CreateConnectorOrg(connector *Connector) error // CreatePipeline creates a pipeline for the @@ -71,6 +75,15 @@ type Client interface { // CreateRepository creates a repository. CreateRepository(org, project string, repo *RepositoryCreateRequest) (*Repository, error) + + // UploadHarnessCodeZip upload zip to harness code for creating repo and other metadata. + UploadHarnessCodeZip(space, zipFileLocation, requestId string, request *types.RepositoriesImportInput) (*types.RepositoriesImportOutput, error) + + // HarnessCodeInviteUser provides all email id to harness code of users which needs to invited. + HarnessCodeInviteUser(space, requestId string, in *types.RepositoryUsersImportInput) (*types.RepositoryUsersImportOutput, error) + + // HarnessCodeCheckImport checks status of import. + HarnessCodeCheckImport(space, requestId string) (*types.RepositoryImportStatus, error) } // WaitHarnessSecretManager blocks until the harness diff --git a/internal/harness/client_impl.go b/internal/harness/client_impl.go index 5e99d5f..a5759e9 100644 --- a/internal/harness/client_impl.go +++ b/internal/harness/client_impl.go @@ -21,11 +21,14 @@ import ( "fmt" "io" "io/ioutil" + "mime/multipart" "net/http" "net/http/httputil" "net/url" "os" "strconv" + + "github.com/harness/harness-migrate/types" ) type client struct { @@ -256,6 +259,47 @@ func (c *client) CreateRepository(org, project string, repo *RepositoryCreateReq return out, nil } +func (c *client) UploadHarnessCodeZip(space, zipFileLocation, requestId string, in *types.RepositoriesImportInput) (*types.RepositoriesImportOutput, error) { + out := new(types.RepositoriesImportOutput) + uri := fmt.Sprintf("%s/gateway/code/api/v1/spaces/%s/zip-import", + c.address, + space, + ) + + if err := c.doMultiPart(uri, "POST", zipFileLocation, requestId, in, out); err != nil { + return nil, err + } + return out, nil +} + +func (c *client) HarnessCodeInviteUser(space, requestId string, in *types.RepositoryUsersImportInput) (*types.RepositoryUsersImportOutput, error) { + out := new(types.RepositoryUsersImportOutput) + uri := fmt.Sprintf("%s/gateway/code/api/v1/spaces/%s/import/%s/users", + c.address, + space, + requestId, + ) + + if err := c.post(uri, in, out); err != nil { + return nil, err + } + return out, nil +} + +func (c *client) HarnessCodeCheckImport(space, requestId string) (*types.RepositoryImportStatus, error) { + out := new(types.RepositoryImportStatus) + uri := fmt.Sprintf("%s/gateway/code/api/v1/spaces/%s/import/%s", + c.address, + space, + requestId, + ) + + if err := c.get(uri, out); err != nil { + return nil, err + } + return out, nil +} + // // http request helper functions // @@ -293,6 +337,19 @@ func (c *client) do(rawurl, method string, in, out interface{}) error { return nil } +// helper function to make an http multipart request +func (c *client) doMultiPart(rawurl, method, filePath, requestId string, in, out interface{}) error { + body, err := c.openMultipart(rawurl, method, filePath, requestId, in) + if err != nil { + return err + } + defer body.Close() + if out != nil { + return json.NewDecoder(body).Decode(out) + } + return nil +} + // helper function to open an http request func (c *client) open(rawurl, method string, in, out interface{}) (io.ReadCloser, error) { uri, err := url.Parse(rawurl) @@ -354,3 +411,91 @@ func (c *client) open(rawurl, method string, in, out interface{}) (io.ReadCloser } return resp.Body, nil } + +// helper function to openMultipart http request +func (c *client) openMultipart(rawurl, method, filepath, requestId string, in interface{}) (io.ReadCloser, error) { + uri, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + req, err := http.NewRequest(method, uri.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("x-api-key", c.token) + req.Header.Set("Accept", "*/*") + req.Header.Set("User-Agent", "harness-migrator") + req.Header.Set("X-Request-ID", requestId) + + // Open the file + file, err := os.Open(filepath) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + defer file.Close() + + // Create a buffer to hold the multipart data + var b bytes.Buffer + writer := multipart.NewWriter(&b) + + // Create a form field and write the file content into it + part, err := writer.CreateFormFile(types.MultiPartFileField, filepath) + if err != nil { + return nil, fmt.Errorf("error creating form file: %w", err) + } + if _, err := io.Copy(part, file); err != nil { + return nil, fmt.Errorf("error copying file: %w", err) + } + + jsonData, derr := json.Marshal(in) + if derr != nil { + return nil, derr + } + + part, err = writer.CreateFormField(types.MultiPartDataField) + if err != nil { + return nil, fmt.Errorf("error creating form field: %w", err) + } + + if _, err := part.Write(jsonData); err != nil { + return nil, fmt.Errorf("error writing JSON field: %w", err) + } + + // Close the writer to finalize the multipart form + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("error closing writer: %w", err) + } + + req.Header.Set("Content-Type", writer.FormDataContentType()) + + // if tracing enabled, dump the request body. + if c.tracing { + dump, _ := httputil.DumpRequest(req, true) + os.Stdout.Write(dump) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + // if tracing enabled, dump the response body. + if c.tracing { + dump, _ := httputil.DumpResponse(resp, true) + os.Stdout.Write(dump) + } + + if resp.StatusCode > 299 { + defer resp.Body.Close() + out, _ := io.ReadAll(resp.Body) + // attempt to unmarshal the error into the + // custom Error structure. + resperr := new(Error) + if jsonerr := json.Unmarshal(out, resperr); jsonerr == nil { + return nil, resperr + } + // else return the error body as a string + return nil, fmt.Errorf("client error %d: %s", resp.StatusCode, string(out)) + } + return resp.Body, nil +} diff --git a/internal/migrate/stash/stash.go b/internal/migrate/stash/stash.go index 900d011..296759a 100644 --- a/internal/migrate/stash/stash.go +++ b/internal/migrate/stash/stash.go @@ -22,6 +22,7 @@ import ( "github.com/harness/harness-migrate/internal/checkpoint" "github.com/harness/harness-migrate/internal/codeerror" "github.com/harness/harness-migrate/internal/common" + "github.com/harness/harness-migrate/internal/gitexporter" "github.com/harness/harness-migrate/internal/tracer" "github.com/harness/harness-migrate/internal/types" @@ -120,7 +121,7 @@ func (e *Export) ListPullRequest( e.tracer.LogError(common.ErrPrList, err) return nil, fmt.Errorf("cannot list pr: %w", err) } - mappedPrs := common.MapPullRequest(prs) + mappedPrs := gitexporter.MapPullRequest(prs) allPrs = append(allPrs, mappedPrs...) err = e.checkpointManager.SaveCheckpoint(checkpointDataKey, allPrs) diff --git a/internal/types/types.go b/internal/types/types.go index e252d24..fab5ef1 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -13,6 +13,7 @@ // limitations under the License. // Package types provides common types for data export and import. + package types import "github.com/drone/go-scm/scm" diff --git a/types/git_exporter.go b/types/git_exporter.go new file mode 100644 index 0000000..1874ea1 --- /dev/null +++ b/types/git_exporter.go @@ -0,0 +1,30 @@ +package types + +import "github.com/drone/go-scm/scm" + +const ( + InfoFileName = "info.json" + PRFolderName = "pr" +) + +type Repository struct { + scm.Repository + Slug string +} + +type PR struct { + scm.PullRequest +} + +type Comments struct { +} + +type RepositoryData struct { + Repository Repository + PullRequestData []*PullRequestData +} + +type PullRequestData struct { + PullRequest PR + Comments []Comments +} diff --git a/types/git_importer.go b/types/git_importer.go new file mode 100644 index 0000000..c5550e5 --- /dev/null +++ b/types/git_importer.go @@ -0,0 +1,61 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package types provides common types for data export and import. + +package types + +const ( + MultiPartFileField = "file" + MultiPartDataField = "data" +) + +type ( + // RepositoriesImportInput is input object for importing repo(s). + RepositoriesImportInput struct { + RequestId string `json:"request_id"` + } + + // RepositoriesImportOutput is output object for importing repo(s). + RepositoriesImportOutput struct { + RequestId string `json:"request_id"` + Users struct { + NotPresent []string `json:"not_present"` + } `json:"users,omitempty"` + } + + // RepositoryUsersImportInput is object for creating/rejecting user invite during repo(s) import. + RepositoryUsersImportInput struct { + Emails []string `json:"emails"` + MapToImporter bool `json:"map_to_importer"` + } + + // RepositoryUsersImportOutput is object for reporting back user import. + RepositoryUsersImportOutput struct { + Success bool `json:"success"` + ErrorMessage string `json:"error_message"` + } + + // RepositoryImportStatus is object for reporting back status of import. + RepositoryImportStatus struct { + Status RepoImportStatus `json:"status"` + } +) + +type RepoImportStatus string + +const ( + RepoImportStatusComplete RepoImportStatus = "complete" + RepoImportStatusInProgress RepoImportStatus = "in_progress" +)