From 70d2eb6d39e896a6923b32d952bfc32b8e1c22f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Fr=C3=B6lich?= Date: Sat, 13 Jul 2024 12:12:20 +0200 Subject: [PATCH 1/2] wip: move to cobra --- cmd/root.go | 24 ++++++++++ cmd/scrape.go | 40 ++++++++++++++++ go.mod | 3 ++ go.sum | 9 ++++ main.go | 116 +---------------------------------------------- scrape/scrape.go | 100 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 178 insertions(+), 114 deletions(-) create mode 100644 cmd/root.go create mode 100644 cmd/scrape.go create mode 100644 scrape/scrape.go diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..f3ee594 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,24 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "starscraper", + Short: "Github Stargazer data getter.", + Long: `Starscraper is a simple application that returns public information for the stargazers of a given Github repo.`, +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + +} diff --git a/cmd/scrape.go b/cmd/scrape.go new file mode 100644 index 0000000..bd6f539 --- /dev/null +++ b/cmd/scrape.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/tobifroe/starscraper/scrape" +) + +// scrapeCmd represents the scrape command +var scrapeCmd = &cobra.Command{ + Use: "scrape", + Short: "Scrapes stargazer data", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + owner := cmd.Flag("owner").Value.String() + repo := cmd.Flag("repo").Value.String() + token := cmd.Flag("token").Value.String() + output := cmd.Flag("output").Value.String() + verbose, _ := cmd.Flags().GetBool("verbose") + scrape.Scrape(token, repo, owner, output, verbose) + }, +} + +func init() { + rootCmd.AddCommand(scrapeCmd) + + scrapeCmd.Flags().String("repo", "", "Repository to scrape") + scrapeCmd.Flags().String("owner", "", "Repository owner") + scrapeCmd.Flags().String("token", "", "Github PAT") + scrapeCmd.Flags().String("output", "output.csv", "Output file") + scrapeCmd.Flags().BoolP("verbose", "v", false, "Verbose output") + + scrapeCmd.MarkFlagRequired("repo") + scrapeCmd.MarkFlagRequired("owner") + +} diff --git a/go.mod b/go.mod index 2bd17f5..0eccb5a 100644 --- a/go.mod +++ b/go.mod @@ -15,9 +15,12 @@ require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/fatih/color v1.7.0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.8 // indirect github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect diff --git a/go.sum b/go.sum index 84202dc..44e753d 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2Qx cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Ptu650= github.com/briandowns/spinner v1.23.1/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= @@ -17,6 +18,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= @@ -25,6 +28,7 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/githubv4 v0.0.0-20231126234147-1cffa1f02456 h1:6dExqsYngGEiixqa1vmtlUd+zbyISilg0Cf3GWVdeYM= github.com/shurcooL/githubv4 v0.0.0-20231126234147-1cffa1f02456/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= github.com/shurcooL/githubv4 v0.0.0-20240120211514-18a1ae0e79dc h1:vH0NQbIDk+mJLvBliNGfcQgUmhlniWBDXC79oRxfZA0= @@ -33,6 +37,10 @@ github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064 h1:RCQBSFx5JrsbH github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -63,5 +71,6 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/Iwark/spreadsheet.v2 v2.0.0-20230915040305-7677e8164883 h1:P76GtA9CSDE7tooNd+JRcG5Qzxq8Y236qtgCBxN8WRM= gopkg.in/Iwark/spreadsheet.v2 v2.0.0-20230915040305-7677e8164883/go.mod h1:AJiLW20RvjD8NFw7OxNQFAWXlvIJeb9TDTGBsfCzFcM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 4a9fe3d..70980a5 100644 --- a/main.go +++ b/main.go @@ -1,119 +1,7 @@ package main -import ( - "context" - "flag" - "fmt" - "os" - "time" - - "github.com/briandowns/spinner" - "github.com/joho/godotenv" - "github.com/shurcooL/githubv4" - "github.com/tobifroe/starscraper/types" - "github.com/tobifroe/starscraper/util" - "golang.org/x/oauth2" -) - -var query struct { - Repository struct { - Description string - Stargazers struct { - TotalCount int - PageInfo struct { - EndCursor githubv4.String - HasNextPage bool - } - Edges []struct { - Node struct { - Email string - Name string - Login string - } - } - } `graphql:"stargazers(first: 100, after: $cursor)"` - } `graphql:"repository(owner: $owner, name: $repo)"` -} +import "github.com/tobifroe/starscraper/cmd" func main() { - - // init Flags - tokenFlag := flag.String("token", "", "Github Token") - repoFlag := flag.String("repo", "", "Github Repo") - ownerFlag := flag.String("owner", "", "Github Repo Owner") - outputFlag := flag.String("output", "output.csv", "Output file name") - debugFlag := flag.Bool("debug", false, "Verbose output for debugging") - flag.Parse() - - err := godotenv.Load() - if err != nil { - panic(err) - } - - variables := map[string]interface{}{ - "repo": githubv4.String(*repoFlag), - "owner": githubv4.String(*ownerFlag), - "cursor": (*githubv4.String)(nil), // Null after argument to get first page. - } - - if *tokenFlag == "" { - *tokenFlag = os.Getenv("GH_TOKEN") - } - - if *repoFlag == "" { - fmt.Println("Please specify a repository.") - return - } - - if *ownerFlag == "" { - fmt.Println("Please specify a repository owner.") - return - } - - if *tokenFlag != "" { - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: *tokenFlag}, - ) - tc := oauth2.NewClient(ctx, ts) - - client := githubv4.NewClient(tc) - - s := spinner.New(spinner.CharSets[9], 100*time.Millisecond) - fmt.Println("Getting stargazers...") - s.Start() - - var allUsers []types.User - for { - err := client.Query(ctx, &query, variables) - if err != nil { - fmt.Println(err) - return - } - for _, v := range query.Repository.Stargazers.Edges { - if v.Node.Email != "" { - allUsers = append(allUsers, types.User{ - Email: v.Node.Email, - Name: v.Node.Name, - Login: v.Node.Login, - }) - } - if v.Node.Email != "" && *debugFlag { - fmt.Printf("%s (%s) - %s\n", v.Node.Name, v.Node.Login, v.Node.Email) - } - } - if !query.Repository.Stargazers.PageInfo.HasNextPage { - break - } - variables["cursor"] = githubv4.NewString(query.Repository.Stargazers.PageInfo.EndCursor) - } - - util.WriteToCSV(allUsers, *outputFlag) - s.Stop() - fmt.Println("Success.") - fmt.Printf("Wrote stargazer data to %s \n", *outputFlag) - - } else { - fmt.Println("No Github token supplied. Either pass the -token flag, set up a .env file or set the GH_TOKEN environment variable.") - } + cmd.Execute() } diff --git a/scrape/scrape.go b/scrape/scrape.go new file mode 100644 index 0000000..868d055 --- /dev/null +++ b/scrape/scrape.go @@ -0,0 +1,100 @@ +package scrape + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/briandowns/spinner" + "github.com/joho/godotenv" + "github.com/shurcooL/githubv4" + "github.com/tobifroe/starscraper/types" + "github.com/tobifroe/starscraper/util" + "golang.org/x/oauth2" +) + +var query struct { + Repository struct { + Description string + Stargazers struct { + TotalCount int + PageInfo struct { + EndCursor githubv4.String + HasNextPage bool + } + Edges []struct { + Node struct { + Email string + Name string + Login string + } + } + } `graphql:"stargazers(first: 100, after: $cursor)"` + } `graphql:"repository(owner: $owner, name: $repo)"` +} + +func Scrape(token string, repo string, owner string, output string, verbose bool) { + + err := godotenv.Load() + if err != nil { + panic(err) + } + + variables := map[string]interface{}{ + "repo": githubv4.String(repo), + "owner": githubv4.String(owner), + "cursor": (*githubv4.String)(nil), // Null after argument to get first page. + } + + if token == "" { + token = os.Getenv("GH_TOKEN") + } + + if token != "" { + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + + client := githubv4.NewClient(tc) + + s := spinner.New(spinner.CharSets[9], 100*time.Millisecond) + fmt.Println("Getting stargazers...") + s.Start() + + var allUsers []types.User + for { + err := client.Query(ctx, &query, variables) + if err != nil { + fmt.Println(err) + return + } + for _, v := range query.Repository.Stargazers.Edges { + if v.Node.Email != "" { + allUsers = append(allUsers, types.User{ + Email: v.Node.Email, + Name: v.Node.Name, + Login: v.Node.Login, + }) + } + if v.Node.Email != "" && verbose { + fmt.Printf("%s (%s) - %s\n", v.Node.Name, v.Node.Login, v.Node.Email) + } + } + if !query.Repository.Stargazers.PageInfo.HasNextPage { + break + } + variables["cursor"] = githubv4.NewString(query.Repository.Stargazers.PageInfo.EndCursor) + } + + util.WriteToCSV(allUsers, output) + s.Stop() + fmt.Println("Success.") + fmt.Printf("Wrote stargazer data to %s \n", output) + + } else { + fmt.Println("No Github token supplied. Either pass the -token flag, set up a .env file or set the GH_TOKEN environment variable.") + } +} From 1a06e8cbc321973fc80cf8f76de758905450e924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Fr=C3=B6lich?= Date: Sat, 13 Jul 2024 13:38:59 +0200 Subject: [PATCH 2/2] chore: rename release.yaml --- .github/workflows/{goreleaser.yaml => release.yaml} | 0 .github/workflows/test.yaml | 4 +--- 2 files changed, 1 insertion(+), 3 deletions(-) rename .github/workflows/{goreleaser.yaml => release.yaml} (100%) diff --git a/.github/workflows/goreleaser.yaml b/.github/workflows/release.yaml similarity index 100% rename from .github/workflows/goreleaser.yaml rename to .github/workflows/release.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9ea562d..ca4b454 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,8 +1,6 @@ name: Go on: - push: - branches: [ "main" ] pull_request: branches: [ "main" ] @@ -15,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.22' - name: Test run: make test