introducting brand new detection mode

README.md

@@ -15,7 +15,7 @@ The outcome is useful for bug bounty hunters, red teamers, and penetration teste
 ## Motivation 
 
 While working on [HunterSuite](https://huntersuite.io), and as part of the job, we are always thinking of something we can automate to make black-box security testing easier. We discussed this idea of creating a multiple platform cloud brute-force hunter.mainly to find open buckets, apps, and databases hosted on the clouds and possibly app behind proxy servers.   
-Here is the list issues we found with previous tools:
+Here is the list issues we tried to fix:
 
 - separated wordlists 
 - lack of proper concurrency 
@@ -94,7 +94,7 @@ After setting up your API key, you are ready to use CloudBrute.
 ██║     ██║     ██║   ██║██║   ██║██║  ██║██╔══██╗██╔══██╗██║   ██║   ██║   ██╔══╝  
 ╚██████╗███████╗╚██████╔╝╚██████╔╝██████╔╝██████╔╝██║  ██║╚██████╔╝   ██║   ███████╗
  ╚═════╝╚══════╝ ╚═════╝  ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝  ╚═╝ ╚═════╝    ╚═╝   ╚══════╝
-                                                V 1.0.3
+                                                V 1.0.7
 usage: CloudBrute [-h|--help] -d|--domain "<value>" -k|--keyword "<value>"
                   -w|--wordlist "<value>" [-c|--cloud "<value>"] [-t|--threads
                   <integer>] [-T|--timeout <integer>] [-p|--proxy "<value>"]

go.mod

@@ -3,6 +3,7 @@ module github.com/0xsha/cloudbrute
 go 1.15
 
 require (
+	github.com/PuerkitoBio/goquery v1.6.0
 	github.com/akamensky/argparse v1.2.2
 	github.com/cheggaaa/pb v1.0.29
 	github.com/ipinfo/go-ipinfo v0.0.0-20200706210721-8b290686e53e
@@ -12,4 +13,5 @@ require (
 	github.com/rs/zerolog v1.19.0
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
+	mvdan.cc/xurls/v2 v2.2.0
 )

go.sum

@@ -1,12 +1,21 @@
+github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94=
+github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/akamensky/argparse v1.2.2 h1:P17T0ZjlUNJuWTPPJ2A5dM1wxarHgHqfYH+AZTo2xQA=
 github.com/akamensky/argparse v1.2.2/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
 github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/ipinfo/go-ipinfo v0.0.0-20200706210721-8b290686e53e h1:oXcx7ATJBnVrELwQ+98cPEK/MeMuh3Ni+nsxDfICpBo=
 github.com/ipinfo/go-ipinfo v0.0.0-20200706210721-8b290686e53e/go.mod h1:U5CBM2XxTg7xTs6PlvaipW3spQYg+RwRMaiBd3fL1/I=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -21,13 +30,16 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg=
 github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
 golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -42,5 +54,11 @@ golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A=
+mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=

internal/cloud.go

@@ -4,11 +4,17 @@ import (
 	"errors"
 	"github.com/ipinfo/go-ipinfo/ipinfo"
 	"github.com/rs/zerolog/log"
+	"io/ioutil"
+	"mvdan.cc/xurls/v2"
 	"net"
+	"net/http"
 	"strings"
 )
 
-func CloudDetect(domain string, key string) (string, error) {
+// next version maybe?
+//func CloudDetectJS()  { }
+
+func CloudDetectIP(domain string, key string) (string, error) {
 
 	authTransport := ipinfo.AuthTransport{Token: key}
 	httpClient := authTransport.Client()
@@ -75,6 +81,76 @@ func CloudDetect(domain string, key string) (string, error) {
 
 }
 
+
+func CloudDetectHTML(domain string, c *Config , providerPath string) (string, error) {
+
+	if !strings.HasPrefix(domain, "https://") {
+
+		domain = "https://" + domain
+	}
+
+	resp, err := http.Get(domain)
+	if err != nil {
+		return "", errors.New("can't connect to domain")
+	}
+
+	detected := ""
+
+	for _, provider := range c.Providers {
+
+		providerConfig, err := InitCloudConfig(provider, providerPath)
+
+		var allUrls []string
+		if err != nil {
+			log.Fatal().Err(err).Msg("Exiting...")
+		}
+
+		// append all urls for each provider
+		for _, itemURL := range providerConfig.APPUrls {
+			allUrls = append(allUrls, itemURL)
+		}
+		for _, itemURL := range providerConfig.StorageUrls {
+			allUrls = append(allUrls, itemURL)
+		}
+		for _, itemURL := range providerConfig.AppRegionUrls {
+			allUrls = append(allUrls, itemURL)
+		}
+		for _, itemURL := range providerConfig.StorageRegionUrls {
+			allUrls = append(allUrls, itemURL)
+		}
+
+		// make a dictionary
+		providersUrl := make(map[string][]string)
+		providersUrl[provider] = allUrls
+
+		// let's make it relaxed and not miss any possibility
+		rxRelaxed := xurls.Relaxed()
+		rep, _ := ioutil.ReadAll(resp.Body)
+		rx := rxRelaxed.FindAllString(string(rep), -1)
+
+		for _, linkItem := range rx {
+
+			for _, values := range providersUrl {
+
+				for _, value := range values {
+
+					if strings.Contains(linkItem, value) {
+						detected = provider
+						//log.Info().Msg(provider+" detected in source")
+						break
+					}
+				}
+			}
+		}
+
+		if detected != "" {
+			return provider, nil
+		}
+	}
+	return "", errors.New("no provider found in HTML")
+
+}
+
 func CheckSupportedCloud(org string, c *Config) (string, error) {
 
 	for _, company := range c.Providers {

internal/config.go

@@ -28,7 +28,7 @@ type CloudConfig struct {
 	Regions     []string `yaml:"regions"`
 	APPUrls     []string `yaml:"app_urls"`
 	StorageUrls []string `yaml:"storage_urls"`
-	RegionUrls  []string `yaml:"region_urls"`
+	//RegionUrls  []string `yaml:"region_urls"`
 	AppRegionUrls  []string `yaml:"app_region_urls"`
 	StorageRegionUrls  []string `yaml:"storage_region_urls"`
 }

internal/utils.go

@@ -58,6 +58,23 @@ func SelectRandomItem(agents []string) string {
 
 }
 
+
+//// thanks https://golangcode.com/how-to-check-if-a-string-is-a-url/
+//func IsValidUrl(toTest string) bool {
+//	_, err := url.ParseRequestURI(toTest)
+//	if err != nil {
+//		return false
+//	}
+//
+//	u, err := url.Parse(toTest)
+//	if err != nil || u.Scheme == "" || u.Host == "" {
+//		return false
+//	}
+//
+//	return true
+//}
+
+
 //func WriteResultsToFile(results []string, output string) {
 //
 //	file, err := os.OpenFile(output+".txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)

main.go

@@ -101,7 +101,7 @@ func main() {
 ██║     ██║     ██║   ██║██║   ██║██║  ██║██╔══██╗██╔══██╗██║   ██║   ██║   ██╔══╝  
 ╚██████╗███████╗╚██████╔╝╚██████╔╝██████╔╝██████╔╝██║  ██║╚██████╔╝   ██║   ███████╗
  ╚═════╝╚══════╝ ╚═════╝  ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝  ╚═╝ ╚═════╝    ╚═╝   ╚══════╝
-						V 1.0.5
+						V 1.0.7
 `
 	if !*quite {
 		_, _ = fmt.Fprintf(os.Stderr, banner)
@@ -167,20 +167,31 @@ func main() {
 
 	} else {
 
-		// Detect the cloud
-		cloud, err = engine.CloudDetect(*domain, apiKey)
+		// Detect the cloud using ip info
+		cloud, err = engine.CloudDetectIP(*domain, apiKey)
 		if err != nil {
-			log.Fatal().Err(err).Msg("Exiting...")
+			log.Error().Err(err).Msg("IP detection failed")
 		}
 
+
 	}
 
 	// Do we support the provider?
 	provider, err := engine.CheckSupportedCloud(cloud, config)
 	if err != nil {
-		log.Fatal().Err(err).Msg("Exiting...")
+		log.Warn().Msg("IP detection failed")
+
+		// Detect the cloud from HTML and JavaScript source codes
+		provider, err = engine.CloudDetectHTML(*domain, config , providerPath)
+
+		if err != nil {
+
+			log.Fatal().Err(err).Msg("Source detection failed as well use -c .")
+		}
+
 	}
 
+
 	log.Info().Msg(provider + " detected")
 
 	urls, err := engine.GenerateMutatedUrls(*wordList, *mode, provider, providerPath, *keyword, config.Environments)