From d975ba3bfd87ca4b28ef61b695a90e5f22f5b95c Mon Sep 17 00:00:00 2001 From: yhy <31311038+yhy0@users.noreply.github.com> Date: Tue, 7 May 2024 22:43:05 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=E4=BD=BF=E7=94=A8=E5=8D=8F=E7=A8=8B?= =?UTF-8?q?=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/mitmproxy/go-mitmproxy.go | 10 +- pkg/mitmproxy/passive.go | 4 +- pkg/mitmproxy/task.go | 24 +- pkg/mode/active.go | 37 ++- pkg/output/SCopilot.go | 44 +++- pkg/output/type.go | 20 +- pkg/task/task.go | 480 +++++++++++++++++----------------- 7 files changed, 338 insertions(+), 281 deletions(-) diff --git a/pkg/mitmproxy/go-mitmproxy.go b/pkg/mitmproxy/go-mitmproxy.go index ccc9900..4f115a3 100644 --- a/pkg/mitmproxy/go-mitmproxy.go +++ b/pkg/mitmproxy/go-mitmproxy.go @@ -7,11 +7,11 @@ package mitmproxy **/ import ( + "github.com/panjf2000/ants/v2" "github.com/yhy0/Jie/conf" "github.com/yhy0/Jie/pkg/mitmproxy/go-mitmproxy/proxy" "github.com/yhy0/Jie/pkg/task" "github.com/yhy0/logging" - "github.com/yhy0/sizedwaitgroup" ) var t *task.Task @@ -33,10 +33,12 @@ func NewMitmproxy() { ScanTask: make(map[string]*task.ScanTask), } - t.Wg = sizedwaitgroup.New(t.Parallelism) + pool, _ := ants.NewPool(t.Parallelism) + t.Pool = pool + defer t.Pool.Release() // 释放协程池 // 先加一，这里会一直阻塞，这样就不会马上退出, 这里要的就是一直阻塞，所以不使用 wg.Done() - t.Wg.Add() + t.WG.Add(1) var err error PassiveProxy, err = proxy.NewProxy(opts) @@ -53,5 +55,5 @@ func NewMitmproxy() { } }() - t.Wg.Wait() + t.WG.Wait() } diff --git a/pkg/mitmproxy/passive.go b/pkg/mitmproxy/passive.go index 44d335f..cd443e4 100644 --- a/pkg/mitmproxy/passive.go +++ b/pkg/mitmproxy/passive.go @@ -44,7 +44,7 @@ func judge(f *proxy.Flow) { flag = funk.Contains(conf.GlobalConfig.Mitmproxy.FilterSuffix, ext) } if !flag { - go distribution(f) + distribution(f) } } } else { @@ -55,7 +55,7 @@ func judge(f *proxy.Flow) { flag = funk.Contains(conf.GlobalConfig.Mitmproxy.FilterSuffix, ext) } if !flag { - go distribution(f) + distribution(f) } } } diff --git a/pkg/mitmproxy/task.go b/pkg/mitmproxy/task.go index f5426ff..994b5cd 100644 --- a/pkg/mitmproxy/task.go +++ b/pkg/mitmproxy/task.go @@ -6,7 +6,7 @@ import ( "github.com/yhy0/logging" "net/url" "strings" - + "github.com/yhy0/Jie/pkg/input" "github.com/yhy0/Jie/pkg/protocols/httpx" "strconv" @@ -25,7 +25,7 @@ func distribution(f *proxy.Flow) { logging.Logger.Errorln(err) return } - + var host string // 有的会带80、443端口号，导致 example.com 和 example.com:80、example.com:443被认为是不同的网站 port := strings.Split(parseUrl.Host, ":") @@ -34,14 +34,14 @@ func distribution(f *proxy.Flow) { } else { host = parseUrl.Host } - + // 使用解码后的，不然有的 js f.Response.Body 直接乱码 var body []byte body, err = f.Response.DecodedBody() if err != nil { body = f.Response.Body } - + // TODO 将 http.Header 转换为 map[string]string 有的重复请求头，这里后面遇到了再优化吧 headerMap := make(map[string]string) for key, values := range f.Request.Header { @@ -50,11 +50,11 @@ func distribution(f *proxy.Flow) { if key == "Set-Cookie" { separator = ";" } - + // 将多个值连接成一个字符串，用逗号分隔 headerMap[key] = strings.Join(values, separator) } - + in := &input.CrawlResult{ Target: f.Request.URL.Host, Url: f.Request.URL.String(), @@ -73,6 +73,14 @@ func distribution(f *proxy.Flow) { RawRequest: requestDump(f.Request), RawResponse: responseDump(f), } - - t.Distribution(in) + + t.WG.Add(1) + go func() { + err := t.Pool.Submit(t.Distribution(in)) + if err != nil { + t.WG.Done() + logging.Logger.Errorf("add distribution err:%v, crawlResult:%v", err, in) + } + }() + // t.Distribution(in) } diff --git a/pkg/mode/active.go b/pkg/mode/active.go index cce4ab8..e9d4f5e 100644 --- a/pkg/mode/active.go +++ b/pkg/mode/active.go @@ -3,9 +3,9 @@ package mode import ( "encoding/json" "fmt" + "github.com/panjf2000/ants/v2" "github.com/projectdiscovery/katana/pkg/output" "github.com/thoas/go-funk" - regexp "github.com/wasilibs/go-re2" "github.com/yhy0/Jie/conf" "github.com/yhy0/Jie/crawler" "github.com/yhy0/Jie/crawler/crawlergo" @@ -22,6 +22,7 @@ import ( "github.com/yhy0/sizedwaitgroup" "net/url" "path" + "regexp" "strings" "time" ) @@ -84,8 +85,9 @@ func Active(target string, fingerprint []string) ([]string, []string) { Wg: sizedwaitgroup.New(3 + 3), } - t.Wg = sizedwaitgroup.New(t.Parallelism) - + pool, _ := ants.NewPool(t.Parallelism) + t.Pool = pool + defer t.Pool.Release() // 释放协程池 // 爬虫前，进行连接性、指纹识别、 waf 探测 resp, err := client.Request(target, "GET", "", nil) if err != nil { @@ -107,7 +109,7 @@ func Active(target string, fingerprint []string) ([]string, []string) { subdomains = Katana(target, wafs, t, fingerprint) } - t.Wg.Wait() + t.WG.Wait() logging.Logger.Debugln("Fingerprints: ", t.Fingerprints) @@ -117,8 +119,6 @@ func Active(target string, fingerprint []string) ([]string, []string) { } func Katana(target string, waf []string, t *task.Task, fingerprint []string) []string { - t.Wg.Add() - defer t.Wg.Done() parseUrl, err := url.Parse(target) if err != nil { logging.Logger.Errorln(err) @@ -132,7 +132,7 @@ func Katana(target string, waf []string, t *task.Task, fingerprint []string) []s curl := strings.ReplaceAll(result.Request.URL, "\\n", "") curl = strings.ReplaceAll(curl, "\\t", "") curl = strings.ReplaceAll(curl, "\\n", "") - parseUrl, err := url.Parse(curl) + parseUrl, err = url.Parse(curl) if err != nil { logging.Logger.Errorln(err) return @@ -158,7 +158,8 @@ func Katana(target string, waf []string, t *task.Task, fingerprint []string) []s } else { host = parseUrl.Host } - resp, err := t.ScanTask[host].Client.Request(result.Request.URL, result.Request.Method, result.Request.Body, result.Request.Headers) + var resp *httpx.Response + resp, err = t.ScanTask[host].Client.Request(result.Request.URL, result.Request.Method, result.Request.Body, result.Request.Headers) if err != nil { logging.Logger.Errorln(err) return @@ -200,7 +201,14 @@ func Katana(target string, waf []string, t *task.Task, fingerprint []string) []s } // 分发扫描任务 - go t.Distribution(crawlResult) + t.WG.Add(1) + go func() { + err := t.Pool.Submit(t.Distribution(crawlResult)) + if err != nil { + t.WG.Done() + logging.Logger.Errorf("add distribution err:%v, crawlResult:%v", err, crawlResult) + } + }() } if conf.GlobalConfig.WebScan.Craw == "k" { @@ -216,8 +224,6 @@ func Katana(target string, waf []string, t *task.Task, fingerprint []string) []s // Crawlergo 运行爬虫, 对爬虫结果进行处理 func Crawlergo(target string, waf []string, t *task.Task, fingerprint []string) []string { - t.Wg.Add() - defer t.Wg.Done() var targets []*model.Request var req model.Request @@ -296,7 +302,14 @@ func Crawlergo(target string, waf []string, t *task.Task, fingerprint []string) } // 分发扫描任务 - go t.Distribution(crawlResult) + t.WG.Add(1) + go func() { + err := t.Pool.Submit(t.Distribution(crawlResult)) + if err != nil { + t.WG.Done() + logging.Logger.Errorf("add distribution err:%v, crawlResult:%v", err, crawlResult) + } + }() } // 开始爬虫任务 diff --git a/pkg/output/SCopilot.go b/pkg/output/SCopilot.go index 65ebb00..70422fd 100644 --- a/pkg/output/SCopilot.go +++ b/pkg/output/SCopilot.go @@ -24,6 +24,13 @@ var DataUpdated = make(chan struct{}) // SCopilot 将数据存储到 SCopilotMessage 中 func SCopilot(host string, data SCopilotData) { + _host := strings.Split(host, ":") + if len(_host) > 1 { + if _host[1] == "80" { + host = _host[0] + } + } + lock.Lock() defer lock.Unlock() // 判断 map 中是否存在，存在的话就 append，不存在的话就创建 @@ -34,30 +41,47 @@ func SCopilot(host string, data SCopilotData) { sort.SliceStable(SCopilotMessage[host].SiteMap, func(i, j int) bool { return compareLinks(SCopilotMessage[host].SiteMap[i], SCopilotMessage[host].SiteMap[j]) }) - + SCopilotMessage[host].Fingerprints = funk.UniqString(append(SCopilotMessage[host].Fingerprints, data.Fingerprints...)) - + for _, v := range data.VulMessage { if funk.Contains(SCopilotMessage[host].VulMessage, v) { continue } SCopilotMessage[host].VulMessage = append(SCopilotMessage[host].VulMessage, v) + + if SCopilotMessage[host].VulPlugin == nil { + SCopilotMessage[host].VulPlugin = make(map[string]int) + } + if SCopilotMessage[host].VulPlugin[v.Plugin] > 0 { + SCopilotMessage[host].VulPlugin[v.Plugin] = SCopilotMessage[host].VulPlugin[v.Plugin] + 1 + } else { + SCopilotMessage[host].VulPlugin[v.Plugin] = 1 + } } - + for _, v := range data.InfoMsg { if funk.Contains(SCopilotMessage[host].InfoMsg, v) { continue } SCopilotMessage[host].InfoMsg = append(SCopilotMessage[host].InfoMsg, v) + if SCopilotMessage[host].InfoPlugin == nil { + SCopilotMessage[host].InfoPlugin = make(map[string]int) + } + if SCopilotMessage[host].InfoPlugin[v.Plugin] > 0 { + SCopilotMessage[host].InfoPlugin[v.Plugin] = SCopilotMessage[host].InfoPlugin[v.Plugin] + 1 + } else { + SCopilotMessage[host].InfoPlugin[v.Plugin] = 1 + } } - + for _, v := range data.PluginMsg { if funk.Contains(SCopilotMessage[host].PluginMsg, v) { continue } SCopilotMessage[host].PluginMsg = append(SCopilotMessage[host].PluginMsg, v) } - + for _, v := range SCopilotLists { if v.Host == host { v.InfoCount = len(SCopilotMessage[host].InfoMsg) @@ -65,7 +89,7 @@ func SCopilot(host string, data SCopilotData) { v.VulnCount = len(SCopilotMessage[host].VulMessage) } } - + SCopilotMessage[host].CollectionMsg.Subdomain = funk.UniqString(append(SCopilotMessage[host].CollectionMsg.Subdomain, data.CollectionMsg.Subdomain...)) SCopilotMessage[host].CollectionMsg.OtherDomain = funk.UniqString(append(SCopilotMessage[host].CollectionMsg.OtherDomain, data.CollectionMsg.OtherDomain...)) SCopilotMessage[host].CollectionMsg.PublicIp = funk.UniqString(append(SCopilotMessage[host].CollectionMsg.PublicIp, data.CollectionMsg.PublicIp...)) @@ -75,14 +99,14 @@ func SCopilot(host string, data SCopilotData) { SCopilotMessage[host].CollectionMsg.Others = funk.UniqString(append(SCopilotMessage[host].CollectionMsg.Others, data.CollectionMsg.Others...)) SCopilotMessage[host].CollectionMsg.Urls = funk.UniqString(append(SCopilotMessage[host].CollectionMsg.Urls, data.CollectionMsg.Urls...)) SCopilotMessage[host].CollectionMsg.Api = funk.UniqString(append(SCopilotMessage[host].CollectionMsg.Api, data.CollectionMsg.Api...)) - + } else { SCopilotMessage[host] = &data SCopilotLists = append(SCopilotLists, &SCopilotList{ Host: host, }) } - + // 通知数据已更新 这样防止没有启动前端界面时，造成阻塞 select { case DataUpdated <- struct{}{}: @@ -102,12 +126,12 @@ func compareLinks(a, b string) bool { } aPathComponents := strings.Split(aURL.Path, "/") bPathComponents := strings.Split(bURL.Path, "/") - + for i := 0; i < len(aPathComponents) && i < len(bPathComponents); i++ { if aPathComponents[i] != bPathComponents[i] { return aPathComponents[i] < bPathComponents[i] } } - + return len(aPathComponents) < len(bPathComponents) } diff --git a/pkg/output/type.go b/pkg/output/type.go index 73aae38..3025f0e 100644 --- a/pkg/output/type.go +++ b/pkg/output/type.go @@ -37,15 +37,17 @@ type VulnData struct { } type SCopilotData struct { - Target string `json:"target"` - Ip string `json:"ip"` - HostNoPort string `json:"host_no_port"` - SiteMap []string `json:"site_map"` - Fingerprints []string `json:"fingerprints"` - VulMessage []VulMessage `json:"vul_message"` - InfoMsg []PluginMsg `json:"info_msg"` - PluginMsg []PluginMsg `json:"plugin_msg"` - CollectionMsg Collection `json:"collection_msg"` + Target string `json:"target"` + Ip string `json:"ip"` + HostNoPort string `json:"host_no_port"` + SiteMap []string `json:"site_map"` + Fingerprints []string `json:"fingerprints"` + VulMessage []VulMessage `json:"vul_message"` + VulPlugin map[string]int `json:"vul_plugin"` + InfoMsg []PluginMsg `json:"info_msg"` + InfoPlugin map[string]int `json:"info_plugin"` + PluginMsg []PluginMsg `json:"plugin_msg"` + CollectionMsg Collection `json:"collection_msg"` } type Collection struct { diff --git a/pkg/task/task.go b/pkg/task/task.go index b978a29..8337a28 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -2,6 +2,7 @@ package task import ( "fmt" + "github.com/panjf2000/ants/v2" regexp "github.com/wasilibs/go-re2" "github.com/yhy0/Jie/conf" "github.com/yhy0/Jie/fingprints" @@ -27,286 +28,293 @@ import ( /** @author: yhy @since: 2023/1/5 - @desc: TODO ~~后期看看有没有必要设计成插件式的，自我感觉没必要，还不如这样写，逻辑简单易懂~~ + @desc: ~~后期看看有没有必要设计成插件式的，自我感觉没必要，还不如这样写，逻辑简单易懂~~ 最终还是插件式比较好 ，哈哈 - todo 漏洞检测逻辑有待优化, 每个插件扫描到漏洞后，需要及时退出，不再进行后续扫描, 插件内部应该设置一个通知，扫描到漏洞即停止 + todo 漏洞检测逻辑有待优化, 每个插件扫描到漏洞后，需要及时退出，不再进行后续扫描, 插件内部应该设置一个通知，扫描到漏洞即停止 **/ type Task struct { - Fingerprints []string // 这个只有主动会使用，被动只会新建一个 task，所以不会用到 - Parallelism int // 一个网站同时扫描的最大 url 个数 - Wg *sizedwaitgroup.SizedWaitGroup // 限制同时运行的任务数量 - ScanTask map[string]*ScanTask - Lock sync.Mutex - WgLock sync.Mutex + Fingerprints []string // 这个只有主动会使用，被动只会新建一个 task，所以不会用到 + Parallelism int // 同时扫描的最大 url 个数 + Pool *ants.Pool // 协程池，目前来看只是用来优化被动扫描，减小被动扫描时的协程创建、销毁的开销 + WG sync.WaitGroup // 等待协程池所有任务结束 + ScanTask map[string]*ScanTask // 存储对目标扫描时的一些状态 + Lock sync.Mutex // 对 Distribution函数中的一些 map 并发操作进行保护 + WgLock sync.Mutex // ScanTask 是一个 map，运行插件时会并发操作，加锁保护 + WgAddLock sync.Mutex // ScanTask 是一个 map，运行插件时会并发操作，加锁保护 } type ScanTask struct { - PerServer map[string]bool // 判断当前目标的 web server 是否扫过 key 为插件名字 - PerFolder map[string]bool // 判断当前目标的目录是否扫过 这里的 key 为插件名_目录名 比如 bbscan_/admin - PocPlugin map[string]bool // 用来 poc 漏洞模块对应的指纹扫描是否扫，poc 模块依托于指纹识别，只有识别到了才会扫描 - Client *httpx.Client // 用来进行请求的 client - Archive bool // 用来判断是否扫描过 - // 限制每个扫描任务同时运行的扫描插件个数 - Wg *sizedwaitgroup.SizedWaitGroup + PerServer map[string]bool // 判断当前目标的 web server 是否扫过 key 为插件名字 + PerFolder map[string]bool // 判断当前目标的目录是否扫过 这里的 key 为插件名_目录名 比如 bbscan_/admin + PocPlugin map[string]bool // 用来 poc 漏洞模块对应的指纹扫描是否扫，poc 模块依托于指纹识别，只有识别到了才会扫描 + Client *httpx.Client // 用来进行请求的 client + Archive bool // 用来判断是否扫描过 + Wg *sizedwaitgroup.SizedWaitGroup // 限制对每个url扫描时同时运行的插件数 } var rex = regexp.MustCompile(`//#\s+sourceMappingURL=(.*\.map)`) var seenRequests sync.Map // 这里主要是为了一些返回包检测类的判断是否识别过，减小开销，扫描类内部会判断是否扫描过 +// DistributionTaskFunc ants 提交任务需要一个无参数的函数 +type DistributionTaskFunc func() + // Distribution 对爬虫结果或者被动发现结果进行任务分发 -func (t *Task) Distribution(in *input.CrawlResult) { - if !conf.NoProgressBar { - atomic.AddInt64(&output.TaskCounter, 1) - } - - t.Wg.Add() - defer func() { - t.Wg.Done() - logging.Logger.Debugln("Done", in.Url) +func (t *Task) Distribution(in *input.CrawlResult) DistributionTaskFunc { + return func() { if !conf.NoProgressBar { - atomic.AddInt64(&output.TaskCompletionCounter, 1) - } - }() - - logging.Logger.Debugln(fmt.Sprintf("[%s] [%s] %s 扫描任务开始", in.UniqueId, in.Method, in.Url)) - // 这些返回包内容检测、指纹识别等因为没有使用检测是否扫描的逻辑，所以会重复检测，造成一定程度的资源消耗，问题应该不大 - // TODO 还没有想好怎么写逻辑，因为一些扫描插件会用到这些结果，搞成插件化的话，就需要控制插件的执行顺序，后续看看吧，目前影响不大 - msg := output.SCopilotData{ - Target: in.Host, - } - - // 并发修改 in t.ScanTask，加锁保证安全 - t.Lock.Lock() - if in.Headers == nil { - in.Headers = make(map[string]string) - } else { - if value, ok := in.Headers["Content-Type"]; ok { - in.ContentType = value + atomic.AddInt64(&output.TaskCounter, 1) } - } - - var hostNoPort string - port := strings.Split(in.Host, ":") - if len(port) > 1 { - hostNoPort = strings.Split(in.Host, ":")[0] - } else { - hostNoPort = in.Host - } - - if t.ScanTask[in.Host] == nil { - t.ScanTask[in.Host] = &ScanTask{ - PerServer: make(map[string]bool), - PerFolder: make(map[string]bool), - PocPlugin: make(map[string]bool), - Client: httpx.NewClient(nil), - // 3: 同时运行 3 个插件，3 供 PreServer、 PreFolder、PerFile这三个函数使用，防止马上退出 所以这里同时运行的插件个数为3-5 个 - // TODO 更优雅的实现方式 - Wg: sizedwaitgroup.New(3 + 3), - } - } - - // cdn 只检测一次 - if !t.ScanTask[in.Host].PerServer["cdnCheck"] { - t.ScanTask[in.Host].PerServer["cdnCheck"] = true - if _, ok := output.IPInfoList[hostNoPort]; !ok { - // cdn 检测 - matched, value, itemType, dnsData := util.CheckCdn(hostNoPort) - var ip string - var allRecords []string - var cdn bool - if dnsData != nil { - ip = strings.Join(dnsData.A, " ") - if len(dnsData.A) > 1 { // 解析出多个 ip ，认为是 cdn - cdn = true - } - allRecords = dnsData.AllRecords - in.Ip = ip - } else { // 这里说明传入的就是 ip - in.Ip = hostNoPort - } - output.IPInfoList[hostNoPort] = &output.IPInfo{ - Ip: ip, - AllRecords: allRecords, - Type: itemType, - Value: value, - Cdn: matched, + + defer func() { + t.WG.Done() + logging.Logger.Debugln("扫描任务结束:", in.Url) + if !conf.NoProgressBar { + atomic.AddInt64(&output.TaskCompletionCounter, 1) } - if itemType == "cdn" || cdn { - in.Cdn = true + }() + + logging.Logger.Debugln(fmt.Sprintf("[%s] [%s] %s 扫描任务开始", in.UniqueId, in.Method, in.Url)) + // 这些返回包内容检测、指纹识别等因为没有使用检测是否扫描的逻辑，所以会重复检测，造成一定程度的资源消耗，问题应该不大 + // TODO 还没有想好怎么写逻辑，因为一些扫描插件会用到这些结果，搞成插件化的话，就需要控制插件的执行顺序，后续看看吧，目前影响不大 + msg := output.SCopilotData{ + Target: in.Host, + } + + // 并发修改 in t.ScanTask，加锁保证安全 + t.Lock.Lock() + if in.Headers == nil { + in.Headers = make(map[string]string) + } else { + if value, ok := in.Headers["Content-Type"]; ok { + in.ContentType = value } - } - } - - msg.HostNoPort = hostNoPort - - // 指纹识别, 插件扫描会用到, 还是放到这里吧，多次识别也不影响 - fingerprints := fingprints.Identify([]byte(in.Resp.Body), in.Resp.Header) - if len(fingerprints) > 0 { - msg.Fingerprints = append(msg.Fingerprints, fingerprints...) - in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) - } - - msg.SiteMap = append(msg.SiteMap, in.Url) - - msg.CollectionMsg = collection.Info(in.Url, hostNoPort, in.Resp.Body, in.ContentType) - - // 请求头中存在 Authorization 时，看看是不是 JWT ，如果是，自动对 JWT 进行爆破 - if v, ok := in.Headers["Authorization"]; ok { - value := strings.Split(v, " ") // 有的 JWT ，是 Xxx Jwt 这种格式 - var jwtString string - if len(value) > 1 { - jwtString = value[1] + + var hostNoPort string + _host := strings.Split(in.Host, ":") + if len(_host) > 1 { + hostNoPort = _host[0] } else { - jwtString = value[0] + hostNoPort = in.Host } - // 一个 jwt 解析爆破一次就好了 - if !jwt.Jwts[jwtString] { - // 首先解析一下，看看是不是 Jwt - _, err := jwt.ParseJWT(jwtString) - if err == nil { // 没有报错，说明解析 Jwt 成功 - msg.Fingerprints = util.RemoveDuplicateElement(append(msg.Fingerprints, "jwt")) - in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) - jwt.Jwts[jwtString] = true - secret := jwt.GenerateSignature() - if secret != "" { - output.OutChannel <- output.VulMessage{ - DataType: "web_vul", - Plugin: "JWT", - VulnData: output.VulnData{ - CreateTime: time.Now().Format("2006-01-02 15:04:05"), - Target: in.Url, - Method: in.Method, - Ip: in.Ip, - Payload: secret, - }, - Level: output.Critical, + + if t.ScanTask[in.Host] == nil { + t.ScanTask[in.Host] = &ScanTask{ + PerServer: make(map[string]bool), + PerFolder: make(map[string]bool), + PocPlugin: make(map[string]bool), + Client: httpx.NewClient(nil), + // 3: 同时运行 3 个插件，3 供 PreServer、 PreFolder、PerFile这三个函数使用，防止马上退出 所以这里同时运行的插件个数为3-5 个 + // TODO 更优雅的实现方式 + Wg: sizedwaitgroup.New(3 + 3), + } + } + + // cdn 只检测一次 + if !t.ScanTask[in.Host].PerServer["cdnCheck"] { + t.ScanTask[in.Host].PerServer["cdnCheck"] = true + if _, ok := output.IPInfoList[hostNoPort]; !ok { + // cdn 检测 + matched, value, itemType, dnsData := util.CheckCdn(hostNoPort) + var ip string + var allRecords []string + var cdn bool + if dnsData != nil { + ip = strings.Join(dnsData.A, " ") + if len(dnsData.A) > 1 { // 解析出多个 ip ，认为是 cdn + cdn = true } + allRecords = dnsData.AllRecords + in.Ip = ip + } else { // 这里说明传入的就是 ip + in.Ip = hostNoPort + } + output.IPInfoList[hostNoPort] = &output.IPInfo{ + Ip: ip, + AllRecords: allRecords, + Type: itemType, + Value: value, + Cdn: matched, } + if itemType == "cdn" || cdn { + in.Cdn = true + } + } } - } - - // 没有检测到 shiro 时，扫描的请求中添加一个 请求头检测一下 Cookie: rememberMe=3 - // TODO 这里不对，添加到这里，返回的数据包并没有经过这里，放到 httpx.request(...) 中 进行指纹检测的话，怎么返回获取指纹？还是说对于这种指纹检测，主动进行一次发包 (不太想使用这种方式) - if !util.InSliceCaseFold("shiro", in.Fingerprints) { - if in.Headers["Cookie"] != "" { - in.Headers["Cookie"] = in.Headers["Cookie"] + ";rememberMe=3" - } else { - in.Headers["Cookie"] = "rememberMe=3" - } - } - t.Lock.Unlock() - - // 更新数据 - output.SCopilot(in.Host, msg) - - sensitive.KeyDetection(in.Url, in.Resp.Body) - - errorMsg := sensitive.PageErrorMessageCheck(in.Url, in.RawRequest, in.Resp.Body) - if len(errorMsg) > 0 { - var res []string - for _, v := range errorMsg { - msg.Fingerprints = append(msg.Fingerprints, v.Type) - res = append(res, v.Text) + + msg.HostNoPort = hostNoPort + + // 指纹识别, 插件扫描会用到, 还是放到这里吧，多次识别也不影响 + fingerprints := fingprints.Identify([]byte(in.Resp.Body), in.Resp.Header) + if len(fingerprints) > 0 { + msg.Fingerprints = append(msg.Fingerprints, fingerprints...) + in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) } - msg.PluginMsg = []output.PluginMsg{ - { - Url: in.Url, - Result: res, - Request: in.RawRequest, - Response: in.RawResponse, - }, + + msg.SiteMap = append(msg.SiteMap, in.Url) + + msg.CollectionMsg = collection.Info(in.Url, hostNoPort, in.Resp.Body, in.ContentType) + + // 请求头中存在 Authorization 时，看看是不是 JWT ，如果是，自动对 JWT 进行爆破 + if v, ok := in.Headers["Authorization"]; ok { + value := strings.Split(v, " ") // 有的 JWT ，是 Xxx Jwt 这种格式 + var jwtString string + if len(value) > 1 { + jwtString = value[1] + } else { + jwtString = value[0] + } + // 一个 jwt 解析爆破一次就好了 + if !jwt.Jwts[jwtString] { + // 首先解析一下，看看是不是 Jwt + _, err := jwt.ParseJWT(jwtString) + if err == nil { // 没有报错，说明解析 Jwt 成功 + msg.Fingerprints = util.RemoveDuplicateElement(append(msg.Fingerprints, "jwt")) + in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) + jwt.Jwts[jwtString] = true + secret := jwt.GenerateSignature() + if secret != "" { + output.OutChannel <- output.VulMessage{ + DataType: "web_vul", + Plugin: "JWT", + VulnData: output.VulnData{ + CreateTime: time.Now().Format("2006-01-02 15:04:05"), + Target: in.Url, + Method: in.Method, + Ip: in.Ip, + Payload: secret, + }, + Level: output.Critical, + } + } + } + } } - } - - // 非 css、js 类进行扫描, 单独进行 判断是否识别过 - if !strings.HasSuffix(in.ParseUrl.Path, ".css") && !strings.HasSuffix(in.ParseUrl.Path, ".js") { - // 插件扫描 - t.Run(in) - // poc 模块依托于指纹识别，只有识别到对应的指纹才会扫描，所以这里就不插件化了 - if conf.Plugin["poc"] { - t.Lock.Lock() - t.ScanTask[in.Host].PocPlugin = pocs_go.PocCheck(in.Fingerprints, in.Target, in.Url, in.Ip, t.ScanTask[in.Host].PocPlugin, t.ScanTask[in.Host].Client) - t.Lock.Unlock() + + // 没有检测到 shiro 时，扫描的请求中添加一个 请求头检测一下 Cookie: rememberMe=3 + // TODO 这里不对，添加到这里，返回的数据包并没有经过这里，放到 httpx.request(...) 中 进行指纹检测的话，怎么返回获取指纹？还是说对于这种指纹检测，主动进行一次发包 (不太想使用这种方式) + if !util.InSliceCaseFold("shiro", in.Fingerprints) { + if in.Headers["Cookie"] != "" { + in.Headers["Cookie"] = in.Headers["Cookie"] + ";rememberMe=3" + } else { + in.Headers["Cookie"] = "rememberMe=3" + } } - } else { - // 下面这些使用去重逻辑，因为扫描结果不会被别的插件用到 - if isScanned(in.UniqueId) { - return + t.Lock.Unlock() + + // 更新数据 + output.SCopilot(in.Host, msg) + + sensitive.KeyDetection(in.Url, in.Resp.Body) + + errorMsg := sensitive.PageErrorMessageCheck(in.Url, in.RawRequest, in.Resp.Body) + if len(errorMsg) > 0 { + var res []string + for _, v := range errorMsg { + msg.Fingerprints = append(msg.Fingerprints, v.Type) + res = append(res, v.Text) + } + msg.PluginMsg = []output.PluginMsg{ + { + Url: in.Url, + Result: res, + Request: in.RawRequest, + Response: in.RawResponse, + }, + } } - if strings.HasSuffix(in.ParseUrl.Path, ".js") { - // 对于 js 这种单独判断是否扫描过，减少消耗 - if strings.HasPrefix(in.Resp.Body, "webpackJsonp(") || strings.Contains(in.Resp.Body, "window[\"webpackJsonp\"]") { + // 非 css、js 类进行扫描, 单独进行 判断是否识别过 + if !strings.HasSuffix(in.ParseUrl.Path, ".css") && !strings.HasSuffix(in.ParseUrl.Path, ".js") { + // 插件扫描 + t.Run(in) + // poc 模块依托于指纹识别，只有识别到对应的指纹才会扫描，所以这里就不插件化了 + if conf.Plugin["poc"] { t.Lock.Lock() - msg.Fingerprints = util.RemoveDuplicateElement(append(msg.Fingerprints, "Webpack")) - in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) + t.ScanTask[in.Host].PocPlugin = pocs_go.PocCheck(in.Fingerprints, in.Target, in.Url, in.Ip, t.ScanTask[in.Host].PocPlugin, t.ScanTask[in.Host].Client) t.Lock.Unlock() } + } else { + // 下面这些使用去重逻辑，因为扫描结果不会被别的插件用到 + if isScanned(in.UniqueId) { + return + } - // 前端 js 中存在 sourcemap 文件，即 xxx.js.map 这种可以使用 sourcemap 等工具还原前端代码 - match := rex.FindStringSubmatch(in.Resp.Body) - if match != nil { - t.Lock.Lock() - msg.Fingerprints = util.RemoveDuplicateElement(append(msg.Fingerprints, "SourceMap")) - in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) - t.Lock.Unlock() - output.OutChannel <- output.VulMessage{ - DataType: "web_vul", - Plugin: "SourceMap", - VulnData: output.VulnData{ - CreateTime: time.Now().Format("2006-01-02 15:04:05"), - Target: in.Url, - }, - Level: output.Low, + if strings.HasSuffix(in.ParseUrl.Path, ".js") { + // 对于 js 这种单独判断是否扫描过，减少消耗 + if strings.HasPrefix(in.Resp.Body, "webpackJsonp(") || strings.Contains(in.Resp.Body, "window[\"webpackJsonp\"]") { + t.Lock.Lock() + msg.Fingerprints = util.RemoveDuplicateElement(append(msg.Fingerprints, "Webpack")) + in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) + t.Lock.Unlock() + } + + // 前端 js 中存在 sourcemap 文件，即 xxx.js.map 这种可以使用 sourcemap 等工具还原前端代码 + match := rex.FindStringSubmatch(in.Resp.Body) + if match != nil { + t.Lock.Lock() + msg.Fingerprints = util.RemoveDuplicateElement(append(msg.Fingerprints, "SourceMap")) + in.Fingerprints = util.RemoveDuplicateElement(append(in.Fingerprints, msg.Fingerprints...)) + t.Lock.Unlock() + output.OutChannel <- output.VulMessage{ + DataType: "web_vul", + Plugin: "SourceMap", + VulnData: output.VulnData{ + CreateTime: time.Now().Format("2006-01-02 15:04:05"), + Target: in.Url, + }, + Level: output.Low, + } } } } - } - - // 更新数据 - output.SCopilot(in.Host, msg) - - go func() { - // 判断 Archive 是否有数据，如果有的话分发至扫描 - if !t.ScanTask[in.Host].Archive && in.Archive != nil { - for k, v := range in.Archive { - response, err := t.ScanTask[in.Host].Client.Request(v, "GET", "", nil) - if err != nil { - continue - } - if response.StatusCode == 200 && !scan_util.IsBlackHtml(response.Body, response.Header["Content-Type"]) { + + // 更新数据 + output.SCopilot(in.Host, msg) + + go func() { + // 判断 Archive 是否有数据，如果有的话分发至扫描 + if !t.ScanTask[in.Host].Archive && in.Archive != nil { + for k, v := range in.Archive { + response, err := t.ScanTask[in.Host].Client.Request(v, "GET", "", nil) + if err != nil { + continue + } parseUrl, err := url.Parse(v) if err != nil { continue } - i := &input.CrawlResult{ - Target: in.Target, - Url: v, - Host: in.Host, - ParseUrl: parseUrl, - UniqueId: k, - Method: "GET", - Resp: &httpx.Response{ - Status: strconv.Itoa(response.StatusCode), - StatusCode: response.StatusCode, - Body: response.Body, - Header: response.Header, - }, - RawRequest: response.RequestDump, - RawResponse: response.ResponseDump, + + if response.StatusCode == 200 && !scan_util.IsBlackHtml(response.Body, response.Header["Content-Type"], parseUrl.Path) { + + i := &input.CrawlResult{ + Target: in.Target, + Url: v, + Host: in.Host, + ParseUrl: parseUrl, + UniqueId: k, + Method: "GET", + Resp: &httpx.Response{ + Status: strconv.Itoa(response.StatusCode), + StatusCode: response.StatusCode, + Body: response.Body, + Header: response.Header, + }, + RawRequest: response.RequestDump, + RawResponse: response.ResponseDump, + } + t.Run(i) } - t.Run(i) } } - } - t.Lock.Lock() - t.ScanTask[in.Host].Archive = true - t.Lock.Unlock() - }() + t.Lock.Lock() + t.ScanTask[in.Host].Archive = true + t.Lock.Unlock() + }() + } } func isScanned(key string) bool {