diff --git a/pkg/scripts/issues/README.md b/pkg/scripts/issues/README.md new file mode 100644 index 0000000000..a410f991f6 --- /dev/null +++ b/pkg/scripts/issues/README.md @@ -0,0 +1,11 @@ +1. To use the script, generate access token here: https://github.com/settings/tokens?type=beta. +2. To get all open issues invoke the [first script](./gh/main.go) setting `SF_TF_SCRIPT_GH_ACCESS_TOKEN`: +```shell + cd gh && SF_TF_SCRIPT_GH_ACCESS_TOKEN= go run . +``` +3. File `issues.json` should be generated in the `gh` directory. This is the input file for the second script. +4. To get process the issues invoke the [second script](./file/main.go): +```shell + cd file && go run . +``` +5. File `issues.csv` should be generated in the `file` directory. This is the CSV which summarizes all the issues we have. diff --git a/pkg/scripts/issues/file/main.go b/pkg/scripts/issues/file/main.go new file mode 100644 index 0000000000..c6aabe851a --- /dev/null +++ b/pkg/scripts/issues/file/main.go @@ -0,0 +1,155 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "os" + "regexp" + "slices" + "strconv" + "strings" + "time" + + i "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/scripts/issues" +) + +func main() { + issues := loadIssues() + processedIssues := processIssues(issues) + saveCsv(processedIssues) +} + +func loadIssues() []i.Issue { + bytes, err := os.ReadFile("../gh/issues.json") + if err != nil { + panic(err) + } + var issues []i.Issue + err = json.Unmarshal(bytes, &issues) + if err != nil { + panic(err) + } + return issues +} + +func processIssues(issues []i.Issue) []ProcessedIssue { + processedIssues := make([]ProcessedIssue, 0) + for idx, issue := range issues { + fmt.Printf("Processing issue (%d): %d\n", idx+1, issue.Number) + labels := make([]string, 0) + for _, label := range issue.Labels { + labels = append(labels, label.Name) + } + providerVersion, providerVersionMinor := getProviderVersion(issue) + terraformVersion := getTerraformVersion(issue) + processed := ProcessedIssue{ + ID: issue.Number, + URL: issue.HtmlUrl, + NamedURL: fmt.Sprintf(`=HYPERLINK("%s","#%d")`, issue.HtmlUrl, issue.Number), + Title: issue.Title, + ProviderVersion: providerVersion, + ProviderVersionMinor: providerVersionMinor, + TerraformVersion: terraformVersion, + IsBug: slices.Contains(labels, "bug"), + IsFeatureRequest: slices.Contains(labels, "feature-request"), + CommentsCount: issue.Comments, + ReactionsCount: issue.Reactions.TotalCount, + CreatedAt: issue.CreatedAt, + Labels: labels, + } + processedIssues = append(processedIssues, processed) + } + return processedIssues +} + +/* + * For newer issues it should be where (...) are: + * ### Terraform CLI and Provider Versions (...) ### Terraform Configuration + * For older issues it should be where (...) are: + * **Provider Version** (...) **Terraform Version** + */ +func getProviderVersion(issue i.Issue) (string, string) { + oldRegex := regexp.MustCompile(`\*\*Provider Version\*\*\s*([[:ascii:]]*)\s*\*\*Terraform Version\*\*`) + matches := oldRegex.FindStringSubmatch(issue.Body) + if len(matches) == 0 { + return "NONE", "" + } else { + versionRegex := regexp.MustCompile(`v?\.?(\d+\.(\d+)(.\d+)?)`) + vMatches := versionRegex.FindStringSubmatch(matches[1]) + if len(vMatches) == 0 { + return "NONE", "" + } else { + return vMatches[1], vMatches[2] + } + } +} + +/* + * For newer issues it should be where (...) are: + * ### Terraform CLI and Provider Versions (...) ### Terraform Configuration + * For older issues it should be where (...) are: + * **Terraform Version** (...) **Describe the bug** + */ +func getTerraformVersion(issue i.Issue) string { + oldRegex := regexp.MustCompile(`\*\*Terraform Version\*\*\s*([[:ascii:]]*)\s*\*\*Describe the bug\*\*`) + matches := oldRegex.FindStringSubmatch(issue.Body) + if len(matches) == 0 { + return "NONE" + } else { + versionRegex := regexp.MustCompile(`v?\.?(\d+\.(\d+)(.\d+)?)`) + vMatches := versionRegex.FindStringSubmatch(matches[1]) + if len(vMatches) == 0 { + return "NONE" + } else { + return vMatches[1] + } + } +} + +func saveCsv(issues []ProcessedIssue) { + file, err := os.Create("issues.csv") + if err != nil { + panic(err) + } + defer file.Close() + w := csv.NewWriter(file) + w.Comma = ';' + + data := make([][]string, 0, len(issues)) + for _, issue := range issues { + row := []string{ + // strconv.Itoa(issue.ID), + // issue.URL, + issue.NamedURL, + issue.Title, + issue.ProviderVersion, + issue.ProviderVersionMinor, + issue.TerraformVersion, + strconv.FormatBool(issue.IsBug), + strconv.FormatBool(issue.IsFeatureRequest), + strconv.Itoa(issue.CommentsCount), + strconv.Itoa(issue.ReactionsCount), + issue.CreatedAt.Format(time.DateOnly), + strings.Join(issue.Labels, "|"), + } + data = append(data, row) + } + _ = w.WriteAll(data) +} + +type ProcessedIssue struct { + ID int + URL string + NamedURL string + Title string + ProviderVersion string + ProviderVersionMinor string + TerraformVersion string + IsBug bool + IsFeatureRequest bool + CommentsCount int + ReactionsCount int + CreatedAt time.Time + Labels []string +} diff --git a/pkg/scripts/issues/gh/main.go b/pkg/scripts/issues/gh/main.go new file mode 100644 index 0000000000..a608b74968 --- /dev/null +++ b/pkg/scripts/issues/gh/main.go @@ -0,0 +1,102 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" + + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/scripts/issues" +) + +func main() { + accessToken := getAccessToken() + issues := fetchAllIssues(accessToken) + saveIssues(issues) +} + +func getAccessToken() string { + token := os.Getenv("SF_TF_SCRIPT_GH_ACCESS_TOKEN") + if token == "" { + panic(errors.New("GitHub access token missing")) + } + return token +} + +func fetchAllIssues(token string) []issues.Issue { + client := &http.Client{} + allIssues := make([]issues.Issue, 0) + moreIssues := true + page := 1 + for moreIssues { + fmt.Printf("Running batch %d\n", page) + req := prepareRequest(50, page, token) + bytes := invokeReq(client, req) + batch := getIssuesBatch(bytes) + if len(batch) == 0 { + moreIssues = false + } else { + for _, issue := range batch { + if issue.PullRequest == nil { + allIssues = append(allIssues, issue) + } else { + fmt.Printf("Skipping issue %d, it is a PR\n", issue.Number) + } + } + page++ + } + fmt.Printf("Sleeping for a moment...\n") + time.Sleep(5 * time.Second) + } + return allIssues +} + +func prepareRequest(perPage int, page int, token string) *http.Request { + req, err := http.NewRequest("GET", "https://api.github.com/repos/Snowflake-Labs/terraform-provider-snowflake/issues", nil) + if err != nil { + panic(err) + } + q := req.URL.Query() + q.Add("per_page", strconv.Itoa(perPage)) + q.Add("page", strconv.Itoa(page)) + q.Add("state", "open") + req.URL.RawQuery = q.Encode() + req.Header.Add("Accept", "application/vnd.github+json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + fmt.Printf("Prepared URL: %s\n", req.URL.String()) + return req +} + +func invokeReq(client *http.Client, req *http.Request) []byte { + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + return bodyBytes +} + +func getIssuesBatch(bytes []byte) []issues.Issue { + var issues []issues.Issue + err := json.Unmarshal(bytes, &issues) + if err != nil { + panic(err) + } + return issues +} + +func saveIssues(issues []issues.Issue) { + bytes, err := json.Marshal(issues) + if err != nil { + panic(err) + } + _ = os.WriteFile("issues.json", bytes, 0o600) +} diff --git a/pkg/scripts/issues/model.go b/pkg/scripts/issues/model.go new file mode 100644 index 0000000000..dbf41bff0c --- /dev/null +++ b/pkg/scripts/issues/model.go @@ -0,0 +1,30 @@ +package issues + +import "time" + +type Issue struct { + HtmlUrl string `json:"html_url"` + Number int `json:"number"` + Title string `json:"title"` + Labels []Label `json:"labels"` + State string `json:"state"` + Comments int `json:"comments"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Body string `json:"body"` + Reactions Reactions `json:"reactions"` + PullRequest *PullRequest `json:"pull_request"` +} + +type Label struct { + Url string `json:"url"` + Name string `json:"name"` +} + +type Reactions struct { + TotalCount int `json:"total_count"` +} + +type PullRequest struct { + HtmlUrl string `json:"html_url"` +}