增加导入Naabu和Httpx扫描结果导入到平台，以方便在内网渗透时的信息收集和协同；支持IP任务时跳过端口扫描，读取资产已探测的全部端口…

…进行指纹和信息收集；去除效果不太用好的Wappalyzer功能；

README.md

@@ -15,7 +15,7 @@ Nemo是用来进行自动化信息收集的一个简单平台，通过集成常
 - IP归属地（纯真离线数据）
 - 自定义IP归属地、Service、蜜罐
 - 导入本地的Masscan、Nmap端口扫描结果
-- 导入[fscan](https://github.com/shadow1ng/fscan)扫描结果（适用于内网渗透的资产信息收集）
+- 导入[fscan](https://github.com/shadow1ng/fscan)、[Naabu]( https://github.com/projectdiscovery/naabu)、[Httpx]( https://github.com/projectdiscovery/httpx)扫描结果（适用于内网渗透的资产信息收集）
 - CDN识别
 
 ### 2、域名资产
@@ -29,7 +29,7 @@ Nemo是用来进行自动化信息收集的一个简单平台，通过集成常
 
 - [HTTPX](https://github.com/projectdiscovery/httpx) 
 - [ScreenShot](https://github.com/chromedp/chromedp) （调用chrome headless）
-- [Wappalyzer](https://github.com/AliasIO/Wappalyzer) （基于[webanalyze](https://github.com/rverton/webanalyze) 代码，可[自定义指纹规则](thirdparty/wappalyzer/technologies_custom.json)）
+- ~~[Wappalyzer](https://github.com/AliasIO/Wappalyzer) （基于[webanalyze](https://github.com/rverton/webanalyze) 代码，可[自定义指纹规则](thirdparty/wappalyzer/technologies_custom.json)）~~
 - [ObserverWard](https://github.com/0x727/ObserverWard_0x727)  (指纹信息来源于https://github.com/0x727/FingerprintHub)
 - IconHash（基于[mat/besticon](github.com/mat/besticon)和[Becivells/iconhash](github.com/Becivells/iconhash)项目）
 
@@ -110,6 +110,7 @@ Tested on [ubuntu18.04/20.04 LTS](docs/install_linux.md)、[macOS](docs/install_
 
 ## 版本更新
 
+- 2.5.3：2022-7-20，增加导入Naabu和Httpx扫描结果导入到平台，以方便在内网渗透时的信息收集和协同；支持IP任务时跳过端口扫描，读取资产已探测的全部端口进行指纹和信息收集；去除效果不太用好的Wappalyzer功能；修复ObserverWard设置工作目录以正确加载指纹库，升级到2022年7月最新版本。
 - 2.5.2：2022-7-14，增加域名的Whois查询。注意：由2.5.1版本升级时，需在thirdparty中增加whois目录用于存放whois查询的缓存信息。
 - 2.5.1：2022-6-12，更新beego框架至v2.0.3，sessionOn属性修改至app.conf中，更新chromedp至v0.8.2；增加version.txt，记录每次更新后的版本号并在web的“系统设置”-“配置管理”中显示当前的version；导入fscan扫描结果时将结果中的poc-yaml-*漏洞加入到vulnerability中。注意：由2.5.0版本升级时，需在conf/app.conf中增加对应的sessionOn属性。
 - 2.5.0：2022-5-30，新增定时任务执行；Docker的Ubuntun升级为20.04LTS、调整Docker时区；新增webfiles路径映射，统一处理server的web目录访问，取消原screenshot、taskresult与imageicon目录映射；注意：本次累积升级有较大的改动，由v2.4.21升级需导入task.sql与task_cron.sql，并修改app.conf与server.yml中staticdir字段。

pkg/comm/service.go

@@ -277,10 +277,13 @@ func (s *Service) NewTask(ctx context.Context, args *NewTaskArgs, replay *string
 // LoadOpenedPort 读取指定IP已开放的全部端口
 func (s *Service) LoadOpenedPort(ctx context.Context, args *string, replay *string) error {
 	var resultIPAndPort []string
-	ipDb := db.Ip{}
-	portDb := db.Port{}
+
 	ips := strings.Split(*args, ",")
 	for _, ip := range ips {
+		//Fix Bug：
+		//每次重新初始化数据库对象
+		ipDb := db.Ip{}
+		portDb := db.Port{}
 		host := utils.HostStrip(ip)
 		// 如果不是有效的IP（可能是域名）则直接返回原来的目标）
 		if utils.CheckIPV4(host) == false {

pkg/task/domainscan/result.go

@@ -25,7 +25,6 @@ type Config struct {
 	IsIPPortScan       bool   `json:"portscan"`
 	IsIPSubnetPortScan bool   `json:"subnetPortscan"`
 	IsScreenshot       bool   `json:"screenshot"`
-	IsWappalyzer       bool   `json:"wappalyzer"`
 	IsFingerprintHub   bool   `json:"fingerprinthub"`
 	IsIconHash         bool   `json:"iconhash"`
 	PortTaskMode       int    `json:"portTaskMode"`

pkg/task/pocscan/result.go

@@ -8,10 +8,10 @@ import (
 )
 
 type Config struct {
-	Target         string `json:"target"`
-	PocFile        string `json:"pocFile"`
-	CmdBin         string `json:"cmdBin"`
-	LoadOpenedPort bool   `json:"loadOpenedPort"`
+	Target           string `json:"target"`
+	PocFile          string `json:"pocFile"`
+	CmdBin           string `json:"cmdBin"`
+	IsLoadOpenedPort bool   `json:"loadOpenedPort"`
 }
 
 type Result struct {

pkg/task/portscan/result.go

@@ -24,10 +24,11 @@ type Config struct {
 	IsHttpx          bool   `json:"httpx"`
 	IsWhatWeb        bool   `json:"whatweb"`
 	IsScreenshot     bool   `json:"screenshot"`
-	IsWappalyzer     bool   `json:"wappalyzer"`
 	IsFingerprintHub bool   `json:"fingerprinthub"`
 	IsIconHash       bool   `json:"iconhash"`
 	CmdBin           string `json:"cmdBin"`
+	IsLoadOpenedPort bool   `json:"loadOpenedPort"`
+	IsPortscan       bool   `json:"isPortscan"`
 }
 
 // PortAttrResult 端口属性结果

pkg/task/runner/config.go

@@ -17,13 +17,13 @@ type PortscanRequestParam struct {
 	IsPing           bool   `form:"ping"`
 	ExcludeIP        string `form:"exclude"`
 	IsScreenshot     bool   `form:"screenshot"`
-	IsWappalyzer     bool   `form:"wappalyzer"`
 	IsFingerprintHub bool   `form:"fingerprinthub"`
 	IsIconHash       bool   `form:"iconhash"`
 	TaskMode         int    `form:"taskmode"`
 	IsTaskCron       bool   `form:"taskcron" json:"-"`
 	TaskCronRule     string `form:"cronrule" json:"-"`
 	TaskCronComment  string `form:"croncomment" json:"-"`
+	IsLoadOpenedPort bool   `form:"load_opened_port"`
 }
 
 type DomainscanRequestParam struct {
@@ -43,7 +43,6 @@ type DomainscanRequestParam struct {
 	IsScreenshot     bool   `form:"screenshot"`
 	IsICPQuery       bool   `form:"icpquery"`
 	IsWhoisQuery     bool   `form:"whoisquery"`
-	IsWappalyzer     bool   `form:"wappalyzer"`
 	IsFingerprintHub bool   `form:"fingerprinthub"`
 	IsIconHash       bool   `form:"iconhash"`
 	TaskMode         int    `form:"taskmode"`
@@ -63,7 +62,7 @@ type PocscanRequestParam struct {
 	NucleiPocFile    string `form:"nuclei_poc_file"`
 	IsDirsearch      bool   `form:"dirsearch"`
 	DirsearchExtName string `form:"ext"`
-	LoadOpenedPort   bool   `form:"load_opened_port"`
+	IsLoadOpenedPort bool   `form:"load_opened_port"`
 	IsTaskCron       bool   `form:"taskcron" json:"-"`
 	TaskCronRule     string `form:"cronrule" json:"-"`
 	TaskCronComment  string `form:"croncomment" json:"-"`

pkg/task/runner/run.go

@@ -28,10 +28,8 @@ func StartPortScanTask(req PortscanRequestParam, cronTaskId string) (taskId stri
 	for _, t := range targets {
 		for _, p := range ports {
 			// 端口扫描
-			if req.IsPortScan {
-				if taskId, err = doPortscan(cronTaskId, t, p, req); err != nil {
-					return
-				}
+			if taskId, err = doPortscan(cronTaskId, t, p, req); err != nil {
+				return
 			}
 			// IP归属地：如果有端口执行任务，则IP归属地任务在端口扫描中执行，否则单独执行
 			if !req.IsPortScan && req.IsIPLocation {
@@ -41,19 +39,19 @@ func StartPortScanTask(req PortscanRequestParam, cronTaskId string) (taskId stri
 			}
 			// FOFA
 			if req.IsFofa {
-				if taskId, err = doOnlineAPISearch(cronTaskId, "fofa", t, &req.OrgId, req.IsIPLocation, req.IsHttpx, req.IsWappalyzer, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
+				if taskId, err = doOnlineAPISearch(cronTaskId, "fofa", t, &req.OrgId, req.IsIPLocation, req.IsHttpx, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
 					return
 				}
 			}
 			// Quake
 			if req.IsQuake {
-				if taskId, err = doOnlineAPISearch(cronTaskId, "quake", t, &req.OrgId, req.IsIPLocation, req.IsHttpx, req.IsWappalyzer, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
+				if taskId, err = doOnlineAPISearch(cronTaskId, "quake", t, &req.OrgId, req.IsIPLocation, req.IsHttpx, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
 					return
 				}
 			}
 			// Hunter
 			if req.IsHunter {
-				if taskId, err = doOnlineAPISearch(cronTaskId, "hunter", t, &req.OrgId, req.IsIPLocation, req.IsHttpx, req.IsWappalyzer, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
+				if taskId, err = doOnlineAPISearch(cronTaskId, "hunter", t, &req.OrgId, req.IsIPLocation, req.IsHttpx, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
 					return
 				}
 			}
@@ -134,17 +132,17 @@ func StartDomainScanTask(req DomainscanRequestParam, cronTaskId string) (taskId
 			}
 		}
 		if req.IsFofa {
-			if taskId, err = doOnlineAPISearch(cronTaskId, "fofa", t, &req.OrgId, true, req.IsHttpx, req.IsWappalyzer, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
+			if taskId, err = doOnlineAPISearch(cronTaskId, "fofa", t, &req.OrgId, true, req.IsHttpx, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
 				return
 			}
 		}
 		if req.IsQuake {
-			if taskId, err = doOnlineAPISearch(cronTaskId, "quake", t, &req.OrgId, true, req.IsHttpx, req.IsWappalyzer, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
+			if taskId, err = doOnlineAPISearch(cronTaskId, "quake", t, &req.OrgId, true, req.IsHttpx, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
 				return
 			}
 		}
 		if req.IsHunter {
-			if taskId, err = doOnlineAPISearch(cronTaskId, "hunter", t, &req.OrgId, true, req.IsHttpx, req.IsWappalyzer, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
+			if taskId, err = doOnlineAPISearch(cronTaskId, "hunter", t, &req.OrgId, true, req.IsHttpx, req.IsFingerprintHub, req.IsScreenshot, req.IsIconHash); err != nil {
 				return
 			}
 		}
@@ -171,31 +169,31 @@ func StartPocScanTask(req PocscanRequestParam, cronTaskId string) (taskId string
 		}
 	}
 	if req.IsPocsuiteVerify && req.PocsuitePocFile != "" {
-		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.PocsuitePocFile, CmdBin: "pocsuite", LoadOpenedPort: req.LoadOpenedPort}
+		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.PocsuitePocFile, CmdBin: "pocsuite", IsLoadOpenedPort: req.IsLoadOpenedPort}
 		configJSON, _ := json.Marshal(config)
 		taskId, err = serverapi.NewTask("pocsuite", string(configJSON), cronTaskId)
 		if err != nil {
 			return
 		}
 	}
 	if req.IsXrayVerify && req.XrayPocFile != "" {
-		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.XrayPocFile, CmdBin: "xray", LoadOpenedPort: req.LoadOpenedPort}
+		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.XrayPocFile, CmdBin: "xray", IsLoadOpenedPort: req.IsLoadOpenedPort}
 		configJSON, _ := json.Marshal(config)
 		taskId, err = serverapi.NewTask("xray", string(configJSON), cronTaskId)
 		if err != nil {
 			return
 		}
 	}
 	if req.IsNucleiVerify && req.NucleiPocFile != "" {
-		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.NucleiPocFile, CmdBin: "nuclei", LoadOpenedPort: req.LoadOpenedPort}
+		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.NucleiPocFile, CmdBin: "nuclei", IsLoadOpenedPort: req.IsLoadOpenedPort}
 		configJSON, _ := json.Marshal(config)
 		taskId, err = serverapi.NewTask("nuclei", string(configJSON), cronTaskId)
 		if err != nil {
 			return
 		}
 	}
 	if req.IsDirsearch && req.DirsearchExtName != "" {
-		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.DirsearchExtName, CmdBin: "dirsearch", LoadOpenedPort: req.LoadOpenedPort}
+		config := pocscan.Config{Target: strings.Join(targetList, ","), PocFile: req.DirsearchExtName, CmdBin: "dirsearch", IsLoadOpenedPort: req.IsLoadOpenedPort}
 		configJSON, _ := json.Marshal(config)
 		taskId, err = serverapi.NewTask("dirsearch", string(configJSON), cronTaskId)
 		if err != nil {
@@ -219,10 +217,11 @@ func doPortscan(cronTaskId string, target string, port string, req PortscanReque
 		IsHttpx:          req.IsHttpx,
 		IsWhatWeb:        req.IsWhatweb,
 		IsScreenshot:     req.IsScreenshot,
-		IsWappalyzer:     req.IsWappalyzer,
 		IsFingerprintHub: req.IsFingerprintHub,
 		IsIconHash:       req.IsIconHash,
 		CmdBin:           req.CmdBin,
+		IsPortscan:       req.IsPortScan,
+		IsLoadOpenedPort: req.IsLoadOpenedPort,
 	}
 	if req.CmdBin == "" {
 		config.CmdBin = conf.GlobalWorkerConfig().Portscan.Cmdbin
@@ -268,7 +267,6 @@ func doBatchScan(cronTaskId string, target string, port string, req PortscanRequ
 		IsHttpx:          req.IsHttpx,
 		IsWhatWeb:        req.IsWhatweb,
 		IsScreenshot:     req.IsScreenshot,
-		IsWappalyzer:     req.IsWappalyzer,
 		IsFingerprintHub: req.IsFingerprintHub,
 		IsIconHash:       req.IsIconHash,
 		CmdBin:           "masscan",
@@ -316,7 +314,6 @@ func doDomainscan(cronTaskId string, target string, req DomainscanRequestParam)
 		IsIPPortScan:       req.IsIPPortscan,
 		IsIPSubnetPortScan: req.IsSubnetPortscan,
 		IsScreenshot:       req.IsScreenshot,
-		IsWappalyzer:       req.IsWappalyzer,
 		IsFingerprintHub:   req.IsFingerprintHub,
 		IsIconHash:         req.IsIconHash,
 		PortTaskMode:       req.PortTaskMode,
@@ -340,13 +337,12 @@ func doDomainscan(cronTaskId string, target string, req DomainscanRequestParam)
 }
 
 // doOnlineAPISearch Fofa,hunter,quaker的查询
-func doOnlineAPISearch(cronTaskId string, apiName string, target string, orgId *int, isIplocation, isHttp, isWappalyzer, isFingerprintHub, isScreenshot bool, isIconHash bool) (taskId string, err error) {
+func doOnlineAPISearch(cronTaskId string, apiName string, target string, orgId *int, isIplocation, isHttp, isFingerprintHub, isScreenshot bool, isIconHash bool) (taskId string, err error) {
 	config := onlineapi.OnlineAPIConfig{
 		Target:           target,
 		OrgId:            orgId,
 		IsIPLocation:     isIplocation,
 		IsHttpx:          isHttp,
-		IsWappalyzer:     isWappalyzer,
 		IsFingerprintHub: isFingerprintHub,
 		IsScreenshot:     isScreenshot,
 		IsIconHash:       isIconHash,

pkg/task/workerapi/domainscan.go

@@ -121,6 +121,8 @@ func doPortScan(config domainscan.Config, resultDomainScan *domainscan.Result) {
 				IsScreenshot:     config.IsScreenshot,
 				IsFingerprintHub: config.IsFingerprintHub,
 				IsIconHash:       config.IsIconHash,
+				IsLoadOpenedPort: false, //只扫描当前结果
+				IsPortscan:       true,
 			}
 			configPortScanJSON, _ := json.Marshal(configPortScan)
 			// 创建端口扫描任务

pkg/task/workerapi/fingerprint.go

@@ -21,11 +21,6 @@ func DoIPFingerPrint(config portscan.Config, resultPortScan *portscan.Result) {
 		httpx.ResultPortScan = *resultPortScan
 		httpx.Do()
 	}
-	if config.IsWappalyzer {
-		wappalyzer := fingerprint.NewWappalyzer()
-		wappalyzer.ResultPortScan = *resultPortScan
-		wappalyzer.Do()
-	}
 	if config.IsFingerprintHub {
 		fp := fingerprint.NewFingerprintHub()
 		fp.ResultPortScan = *resultPortScan
@@ -49,11 +44,6 @@ func DoDomainFingerPrint(config domainscan.Config, resultDomainScan *domainscan.
 		whatweb.ResultDomainScan = *resultDomainScan
 		whatweb.Do()
 	}
-	if config.IsWappalyzer {
-		wappalyzer := fingerprint.NewWappalyzer()
-		wappalyzer.ResultDomainScan = *resultDomainScan
-		wappalyzer.Do()
-	}
 	if config.IsFingerprintHub {
 		fp := fingerprint.NewFingerprintHub()
 		fp.ResultDomainScan = *resultDomainScan

pkg/task/workerapi/onlineapi.go

@@ -94,7 +94,6 @@ func doFingerAndSave(taskId string, portScanResult *portscan.Result, domainScanR
 		portscanConfig := portscan.Config{
 			IsHttpx:          config.IsHttpx,
 			IsWhatWeb:        config.IsWhatWeb,
-			IsWappalyzer:     config.IsWappalyzer,
 			IsFingerprintHub: config.IsFingerprintHub,
 			IsIconHash:       config.IsIconHash,
 		}
@@ -107,7 +106,6 @@ func doFingerAndSave(taskId string, portScanResult *portscan.Result, domainScanR
 		domainscanConfig := domainscan.Config{
 			IsHttpx:          config.IsHttpx,
 			IsWhatWeb:        config.IsWhatWeb,
-			IsWappalyzer:     config.IsWappalyzer,
 			IsFingerprintHub: config.IsFingerprintHub,
 			IsIconHash:       config.IsIconHash,
 		}

pkg/task/workerapi/pocscan.go

@@ -20,7 +20,7 @@ func PocScan(taskId, configJSON string) (result string, err error) {
 	}
 	x := comm.NewXClient()
 	//读取资产开放端口
-	if config.LoadOpenedPort {
+	if config.IsLoadOpenedPort {
 		err = x.Call(context.Background(), "LoadOpenedPort", &config.Target, &result)
 		if err == nil {
 			config.Target = result

pkg/task/workerapi/portscan.go

@@ -6,7 +6,9 @@ import (
 	"github.com/hanc00l/nemo_go/pkg/comm"
 	"github.com/hanc00l/nemo_go/pkg/logging"
 	"github.com/hanc00l/nemo_go/pkg/task/portscan"
+	"github.com/hanc00l/nemo_go/pkg/utils"
 	"github.com/remeh/sizedwaitgroup"
+	"strconv"
 	"strings"
 )
 
@@ -25,39 +27,70 @@ func PortScan(taskId, configJSON string) (result string, err error) {
 		return FailedTask(err.Error()), err
 	}
 	var resultPortScan portscan.Result
-	// 端口扫描
-	if config.CmdBin == "masnmap" {
-		resultPortScan = doMasscanPlusNmap(config)
-	} else if config.CmdBin == "nmap" {
-		nmap := portscan.NewNmap(config)
-		nmap.Do()
-		resultPortScan = nmap.Result
+	x := comm.NewXClient()
+	//端口扫描：
+	if config.IsPortscan {
+		if config.CmdBin == "masnmap" {
+			resultPortScan = doMasscanPlusNmap(config)
+		} else if config.CmdBin == "nmap" {
+			nmap := portscan.NewNmap(config)
+			nmap.Do()
+			resultPortScan = nmap.Result
+		} else {
+			masscan := portscan.NewMasscan(config)
+			masscan.Do()
+			resultPortScan = masscan.Result
+		}
 	} else {
-		masscan := portscan.NewMasscan(config)
-		masscan.Do()
-		resultPortScan = masscan.Result
+		resultPortScan.IPResult = make(map[string]*portscan.IPResult)
+	}
+	// 读取目标的数据库中已保存的开放端口
+	if config.IsLoadOpenedPort {
+		err = x.Call(context.Background(), "LoadOpenedPort", &config.Target, &result)
+		if err == nil && result != "" {
+			allTargets := strings.Split(result, ",")
+			for _, target := range allTargets {
+				// 必须是ip:port格式
+				dataArray := strings.Split(target, ":")
+				if len(dataArray) != 2 {
+					continue
+				}
+				ip := dataArray[0]
+				port, err := strconv.Atoi(dataArray[1])
+				if utils.CheckIPV4(ip) == false || err != nil {
+					continue
+				}
+				if !resultPortScan.HasIP(ip) {
+					resultPortScan.SetIP(ip)
+				}
+				if !resultPortScan.HasPort(ip, port) {
+					resultPortScan.SetPort(ip, port)
+				}
+			}
+		} else {
+			logging.RuntimeLog.Error(err)
+		}
 	}
 	// IP位置
 	if config.IsIpLocation {
 		doLocation(&resultPortScan)
 	}
 	// 指纹识别
-	DoIPFingerPrint(config,&resultPortScan)
+	DoIPFingerPrint(config, &resultPortScan)
 	// 保存结果
 	resultArgs := comm.ScanResultArgs{
-		TaskID: taskId,
+		TaskID:   taskId,
 		IPConfig: &config,
 		IPResult: resultPortScan.IPResult,
 	}
-	x := comm.NewXClient()
 	err = x.Call(context.Background(), "SaveScanResult", &resultArgs, &result)
 	if err != nil {
 		logging.RuntimeLog.Error(err)
 		return FailedTask(err.Error()), err
 	}
 	// screenshot
 	if config.IsScreenshot {
-		result2 := DoScreenshotAndSave(&resultPortScan,nil)
+		result2 := DoScreenshotAndSave(&resultPortScan, nil)
 		result = strings.Join([]string{result, result2}, ",")
 	}