v1.1.0

thisisiliya · Dec 12, 2023 · 41421eb · 41421eb
1 parent 757bf5b
commit 41421eb
Show file tree

Hide file tree

Showing 18 changed files with 179 additions and 176 deletions.
diff --git a/cmd/custom.go b/cmd/custom.go
@@ -7,7 +7,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/thisisiliya/httpr/pkg/engines"
 	"github.com/thisisiliya/httpr/pkg/request"
-	"github.com/thisisiliya/httpr/pkg/request/validate"
 	"github.com/thisisiliya/httpr/pkg/utils"
 	"golang.org/x/exp/slices"
 )
@@ -16,43 +15,46 @@ var customCmd = &cobra.Command{
 	Use:     "custom",
 	Short:   "custom dork command to scrape",
 	Long:    "engine page(s) scrape by custom dork commands",
-	Example: `httpr custom --command "site:www.google.com inurl:map" --target-host google.com --depth 1`,
+	Example: `httpr custom --command "site:*.hackerone.com inurl:report" --target-host hackerone.com --engine Google --depth 1`,
 	Run:     CustomEnum,
 }
 
 func CustomEnum(_ *cobra.Command, _ []string) {
 
-	var data []request.Data
-
-	ctx, cancel1, cancel2 = utils.Start(root_Proxy, root_Silent)
-	defer cancel1()
-	defer cancel2()
+	utils.Start(root_Silent)
 
 	o.MinDelay = i.root_MinDelay
 	o.MaxDelay = i.root_MaxDelay
+
+	o.Browser = request.Browser(root_Proxy)
+	defer o.Browser.MustClose()
+
 	o.Dork.Domain = i.custom_TargetHost
 	o.Dork.Command = strings.ReplaceAll(i.custom_Command, i.custom_SpiltBy, " ")
 
 	switch strings.ToLower(i.custom_Engine) {
 
 	case "google":
-		o.Engines = append(o.Engines, engines.GoogleURLEncode)
+		o.Engines = []request.Engines{{Engine: engines.GoogleURLEncode, Selector: engines.Google_Selector}}
 
 	case "bing":
-		o.Engines = append(o.Engines, engines.BingURLEncode)
+		o.Engines = []request.Engines{{Engine: engines.BingURLEncode, Selector: engines.Bing_Selector}}
 
 	case "yahoo":
-		o.Engines = append(o.Engines, engines.YahooURLEncode)
+		o.Engines = []request.Engines{{Engine: engines.YahooURLEncode, Selector: engines.Yahoo_Selector}}
+
+	default:
+		o.Engines = []request.Engines{}
 	}
 
+	var data []request.Data
+
 	for o.Dork.Page < i.custom_Depth {
 
-		request.Scrape(&o, &data, &wg, &ctx)
+		data = append(data, *request.Scrape(&o)...)
 		o.Dork.Page++
 	}
 
-	wg.Wait()
-
 	for _, d := range data {
 
 		if CustomValidate(&d) {
@@ -83,7 +85,7 @@ func CustomValidate(d *request.Data) bool {
 
 			if i.root_Verify {
 
-				if validate.Verify(d.URL) {
+				if request.Verify(d.URL) {
 
 					results = append(results, d.URL)
 					return true
@@ -104,7 +106,7 @@ func init() {
 	rootCmd.AddCommand(customCmd)
 
 	customCmd.Flags().StringVarP(&i.custom_Command, "command", "c", "", "dork command to scrape")
-	customCmd.Flags().StringVarP(&i.custom_Engine, "engine", "e", "Google", "target engine to scrape. available engines:Google, Bing, Yahoo")
+	customCmd.Flags().StringVarP(&i.custom_Engine, "engine", "e", "Google", "target engine to scrape. available engines: Google, Bing, Yahoo")
 	customCmd.Flags().StringVarP(&i.custom_TargetHost, "target-host", "t", "", "filter result by host")
 	customCmd.Flags().StringVar(&i.custom_SpiltBy, "split-by", " ", "dork commands split character")
 	customCmd.Flags().IntVar(&i.custom_Depth, "depth", 1, "number of pages to scrape")

diff --git a/cmd/key.go b/cmd/key.go
@@ -6,9 +6,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/thisisiliya/go_utils/errors"
 	"github.com/thisisiliya/go_utils/file"
-	"github.com/thisisiliya/httpr/pkg/engines"
 	"github.com/thisisiliya/httpr/pkg/request"
-	"github.com/thisisiliya/httpr/pkg/request/validate"
 	"github.com/thisisiliya/httpr/pkg/utils"
 	"golang.org/x/exp/slices"
 )
@@ -17,23 +15,19 @@ var keyCmd = &cobra.Command{
 	Use:     "key",
 	Short:   "keywords enumeration for domains",
 	Long:    "keyword(s) enumeration for domain(s)",
-	Example: "httpr key --domain www.google.com --keyword exploit --depth 3",
+	Example: "httpr key --domain hackerone.com --keyword report --depth 3",
 	Run:     KeyExt,
 }
 
 func KeyExt(_ *cobra.Command, _ []string) {
 
-	ctx, cancel1, cancel2 = utils.Start(root_Proxy, root_Silent)
-	defer cancel1()
-	defer cancel2()
+	utils.Start(root_Silent)
 
 	o.MinDelay = i.root_MinDelay
 	o.MaxDelay = i.root_MaxDelay
-	o.Engines = append(o.Engines,
-		engines.GoogleURL,
-		engines.BingURL,
-		engines.YahooURL,
-	)
+
+	o.Browser = request.Browser(root_Proxy)
+	defer o.Browser.MustClose()
 
 	switch {
 
@@ -100,12 +94,10 @@ func KeyEnum() {
 
 	for o.Dork.Page < i.key_Depth {
 
-		request.Scrape(&o, &data, &wg, &ctx)
+		data = append(data, *request.Scrape(&o)...)
 		o.Dork.Page++
 	}
 
-	wg.Wait()
-
 	for _, d := range data {
 
 		if KeyValidate(&d) {
@@ -136,7 +128,7 @@ func KeyValidate(data *request.Data) bool {
 
 			if i.root_Verify {
 
-				if validate.Verify(data.URL) {
+				if request.Verify(data.URL) {
 
 					results = append(results, data.URL)
 					return true
@@ -156,8 +148,6 @@ func init() {
 
 	rootCmd.AddCommand(keyCmd)
 
-	keyCmd.AddGroup()
-
 	keyCmd.Flags().StringVarP(&i.key_Domain, "domain", "d", "", "target domain to search keyword(s)")
 	keyCmd.Flags().StringVar(&i.key_Domains, "domains", "", "target domains file path")
 	keyCmd.Flags().StringVarP(&i.key_Keyword, "keyword", "k", "", "target keyword to search")

diff --git a/cmd/path.go b/cmd/path.go
@@ -7,8 +7,6 @@ import (
 	"github.com/thisisiliya/go_utils/errors"
 	"github.com/thisisiliya/go_utils/file"
 	"github.com/thisisiliya/httpr/pkg/request"
-	"github.com/thisisiliya/httpr/pkg/engines"
-	"github.com/thisisiliya/httpr/pkg/request/validate"
 	"github.com/thisisiliya/httpr/pkg/utils"
 	"golang.org/x/exp/slices"
 )
@@ -23,17 +21,13 @@ var pathCmd = &cobra.Command{
 
 func PathExt(_ *cobra.Command, _ []string) {
 
-	ctx, cancel1, cancel2 = utils.Start(root_Proxy, root_Silent)
-	defer cancel1()
-	defer cancel2()
+	utils.Start(root_Silent)
 
 	o.MinDelay = i.root_MinDelay
 	o.MaxDelay = i.root_MaxDelay
-	o.Engines = append(o.Engines,
-		engines.GoogleURL,
-		engines.BingURL,
-		engines.YahooURL,
-	)
+
+	o.Browser = request.Browser(root_Proxy)
+	defer o.Browser.MustClose()
 
 	switch {
 
@@ -60,12 +54,10 @@ func PathEnum() {
 
 	for o.Dork.Page < i.path_Depth {
 
-		request.Scrape(&o, &data, &wg, &ctx)
+		data = append(data, *request.Scrape(&o)...)
 		o.Dork.Page++
 	}
 
-	wg.Wait()
-
 	for _, d := range data {
 
 		if PathValidate(&d) {
@@ -90,7 +82,7 @@ func PathValidate(data *request.Data) bool {
 
 			if i.root_Verify {
 
-				if validate.Verify(data.URL) {
+				if request.Verify(data.URL) {
 
 					results = append(results, data.Path)
 					return true

diff --git a/cmd/root.go b/cmd/root.go
@@ -1,12 +1,11 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"os"
-	"sync"
 
 	"github.com/spf13/cobra"
+	"github.com/thisisiliya/httpr/pkg/engines"
 	"github.com/thisisiliya/httpr/pkg/request"
 )
 
@@ -17,12 +16,8 @@ var (
 	root_Proxy  string
 	root_Silent bool
 
-	i       Options         // User inputs
-	o       request.Options // Request options
-	wg      sync.WaitGroup
-	ctx     context.Context
-	cancel1 context.CancelFunc
-	cancel2 context.CancelFunc
+	i Options         // User inputs
+	o request.Options // Request options
 
 	rootCmd = &cobra.Command{
 		Use: "httpr",
@@ -33,6 +28,12 @@ var (
 
 func Execute() {
 
+	o.Engines = []request.Engines{
+		{Engine: engines.GoogleURL, Selector: engines.Google_Selector},
+		{Engine: engines.BingURL, Selector: engines.Bing_Selector},
+		{Engine: engines.YahooURL, Selector: engines.Yahoo_Selector},
+	}
+
 	rootCmd.CompletionOptions.DisableDefaultCmd = true
 
 	if err := rootCmd.Execute(); err != nil {
@@ -44,9 +45,9 @@ func Execute() {
 
 func init() {
 
-	rootCmd.PersistentFlags().StringVarP(&root_Proxy, "proxy", "p", "", "proxy url for scraping")
-	rootCmd.PersistentFlags().IntVar(&i.root_MinDelay, "min-delay", 1, "min delay per request")
-	rootCmd.PersistentFlags().IntVar(&i.root_MaxDelay, "max-delay", 10, "max delay per request")
+	rootCmd.PersistentFlags().StringVarP(&root_Proxy, "proxy", "p", "", "proxy or scraping (ip:port)")
+	rootCmd.PersistentFlags().IntVar(&i.root_MinDelay, "min-delay", 1, "min delay per request in sec")
+	rootCmd.PersistentFlags().IntVar(&i.root_MaxDelay, "max-delay", 10, "max delay per request in sec")
 	rootCmd.PersistentFlags().BoolVarP(&i.root_Verify, "verify", "v", false, "verify the result by a request")
 	rootCmd.PersistentFlags().BoolVarP(&root_Silent, "silent", "s", false, "disable printing banner")
 }
diff --git a/cmd/sub.go b/cmd/sub.go
@@ -6,9 +6,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/thisisiliya/go_utils/errors"
 	"github.com/thisisiliya/go_utils/file"
-	"github.com/thisisiliya/httpr/pkg/engines"
 	"github.com/thisisiliya/httpr/pkg/request"
-	"github.com/thisisiliya/httpr/pkg/request/validate"
 	"github.com/thisisiliya/httpr/pkg/utils"
 	"golang.org/x/exp/slices"
 )
@@ -21,25 +19,22 @@ var (
 		Use:     "sub",
 		Short:   "algorithmic subdomain enumeration for domains",
 		Long:    "algorithmic subdomain enumeration for domain(s)",
-		Example: "httpr sub --domain google.com --all",
+		Example: "httpr sub --domain hackerone.com --all",
 		Run:     SubExt,
 	}
 )
 
 func SubExt(_ *cobra.Command, _ []string) {
 
-	ctx, cancel1, cancel2 = utils.Start(root_Proxy, root_Silent)
-	defer cancel1()
-	defer cancel2()
+	utils.Start(root_Silent)
 
 	o.MinDelay = i.root_MinDelay
 	o.MaxDelay = i.root_MaxDelay
+
+	o.Browser = request.Browser(root_Proxy)
+	defer o.Browser.MustClose()
+
 	o.Dork.Wildcard = true
-	o.Engines = append(o.Engines,
-		engines.GoogleURL,
-		engines.BingURL,
-		engines.YahooURL,
-	)
 
 	switch {
 
@@ -116,8 +111,7 @@ func SubScrape() {
 
 	for o.Dork.Page < i.sub_Depth {
 
-		request.Scrape(&o, &data, &wg, &ctx)
-		wg.Wait()
+		data = *request.Scrape(&o)
 
 		for _, d := range data {
 
@@ -139,8 +133,6 @@ func SubScrape() {
 
 		o.Dork.Page++
 	}
-
-	data = []request.Data{}
 }
 
 func SubValidate(d *request.Data) bool {
@@ -151,7 +143,7 @@ func SubValidate(d *request.Data) bool {
 
 			if i.root_Verify {
 
-				if validate.Verify(d.URL) {
+				if request.Verify(d.URL) {
 
 					results = append(results, d.Subdomain)
 					return true

diff --git a/go.mod b/go.mod
@@ -1,23 +1,21 @@
 module github.com/thisisiliya/httpr
 
-go 1.20
+go 1.18
 
 require (
-	github.com/chromedp/cdproto v0.0.0-20231205062650-00455a960d61
-	github.com/chromedp/chromedp v0.9.3
+	github.com/go-rod/rod v0.114.5
+	github.com/go-rod/stealth v0.4.9
 	github.com/spf13/cobra v1.8.0
 	github.com/thisisiliya/go_utils v0.4.2
-	golang.org/x/exp v0.0.0-20231127185646-65229373498e
+	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb
 )
 
 require (
-	github.com/chromedp/sysutil v1.0.0 // indirect
-	github.com/gobwas/httphead v0.1.0 // indirect
-	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.3.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	github.com/ysmood/fetchup v0.2.3 // indirect
+	github.com/ysmood/goob v0.4.0 // indirect
+	github.com/ysmood/got v0.34.1 // indirect
+	github.com/ysmood/gson v0.7.3 // indirect
+	github.com/ysmood/leakless v0.8.0 // indirect
 )