- 2022-10-03 优化了fuzz，http2.0下测试18秒可以完成6万的扫描，同时合并、去除冗余的结果

- 2022-10-03 优化：所有的web扫描前，均做有效检测，避免无效扫描，提升了效率
 2022-10-07

README_CN.md

@@ -207,6 +207,8 @@ priorityNmap=false ./scan4all -tp http -list allOut.txt -v
 more see: <a href=https://github.com/hktalent/scan4all/discussions>discussions</a>
 
 # 变更日志
+- 2022-10-03 优化了fuzz，http2.0下测试18秒可以完成6万的扫描，同时合并、去除冗余的结果
+- 2022-10-03 优化：所有的web扫描前，均做有效检测，避免无效扫描，提升了效率
 - 2022-07-28 为 nuclei 添加 substr、 aes_cbc DSL 函数<a href="https://github.com/projectdiscovery/nuclei/releases/tag/v2.7.7">nuclei v2.7.7</a>
 - 2022-08-03 fixed nuclei Multiple instances cache goroutine leaks PR<a href=https://github.com/projectdiscovery/nuclei/issues/2386>#2386</a>
 - 2022-07-20 fix and PR nuclei <a href=https://github.com/projectdiscovery/nuclei/issues/2301>#2301</a> 并发多实例的bug

brute/dicts/fuzz404.txt

@@ -2,6 +2,7 @@
 403
 <a href="/#/error/404">Found</a>
 404
+404 page not found
 404 Not Found
 404.safedog.cn
 Access Failed

brute/filefuzz.go

@@ -147,6 +147,11 @@ var FileFuzz4Engin = util.EngineFuncFactory(func(evt *models.EventData, args ...
 	util.SendEngineLog(evt, Const.ScanType_WebDirScan, filePaths, fileFuzzTechnologies)
 })
 
+type FuzzData struct {
+	Path *[]string
+	Req  *util.Page
+}
+
 // 重写了fuzz：优化流程、优化算法、修复线程安全bug、增加智能功能
 //  两次  ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
 func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody string) ([]string, []string) {
@@ -195,7 +200,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
 	// 控制 fuzz 线程数
 	var ch = make(chan struct{}, util.Fuzzthreads)
 	// 异步接收结果
-	var async_data = make(chan []string, util.Fuzzthreads*2)
+	var async_data = make(chan *FuzzData, util.Fuzzthreads*2)
 	var async_technologies = make(chan []string, util.Fuzzthreads*2)
 	// 字典长度的 70% 的错误
 	var MaxErrorTimes int32 = int32(float32(len(filedic)) * 0.7)
@@ -209,14 +214,18 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
 	//}()
 	//log.Printf("start fuzz: %s for", u)
 	nStop := 400
+	var lst200 *util.Response
 	go func() {
 		for {
 			select {
 			case <-ctx2.Done():
 				return
 			case x1, ok := <-async_data:
 				if ok {
-					path = append(path, x1...)
+					if lst200 == nil || x1.Req.Resqonse.Body != lst200.Body {
+						path = append(path, (*x1.Path)...)
+					}
+					lst200 = x1.Req.Resqonse
 					if len(path) > nStop {
 						stop() //发停止指令
 						atomic.AddInt32(&errorTimes, MaxErrorTimes)
@@ -237,7 +246,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
 		}
 	}()
 	log.Printf("wait for file fuzz(dicts:%d) %s \r", len(filedic), u)
-	var lst200 *util.Response
+
 	for _, payload := range filedic {
 		// 接收到停止信号
 		if atomic.LoadInt32(&errorTimes) >= MaxErrorTimes {
@@ -315,7 +324,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
 								a11 := ByPass403(&u, &payload, &wg)
 								// 表示 ByPass403 成功了, 结果、控制台输出点什么？
 								if 0 < len(a11) {
-									async_data <- a11
+									async_data <- &FuzzData{Path: &a11, Req: fuzzPage}
 								}
 							}
 							return
@@ -339,7 +348,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
 							path1 = append(path1, *fuzzPage.Url)
 						}
 						if 0 < len(path1) {
-							async_data <- path1
+							async_data <- &FuzzData{Path: &path1, Req: fuzzPage}
 						}
 						if 0 < len(technologies1) {
 							async_technologies <- technologies1
@@ -360,7 +369,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
 	}
 	// 默认情况等待所有结束
 	wg.Wait()
-	log.Printf("fuzz is over: %s\n", u)
+	log.Printf("fuzz is over: %s found:\n%s\n", u, strings.Join(path, "\n"))
 	technologies = util.SliceRemoveDuplicates(technologies)
 	path = util.SliceRemoveDuplicates(path)
 	stop() //发停止指令

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	go.uber.org/ratelimit v0.2.0
 	golang.org/x/net v0.0.0-20221004154528-8021a29435af
-	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec
+	golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875
 )
 
 require (
@@ -101,7 +101,7 @@ require (
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/gorilla/websocket v1.5.0
 	github.com/gosnmp/gosnmp v1.35.0
-	github.com/hktalent/PipelineHttp v0.0.0-20221006120713-f0d2d692285e
+	github.com/hktalent/PipelineHttp v0.0.0-20221007020748-952ec7255bdb
 	github.com/hktalent/goSqlite_gorm v1.1.1
 	github.com/hktalent/jarm-go v0.0.0-20220918133110-7801447b6267
 	github.com/huin/asn1ber v0.0.0-20120622192748-af09f62e6358
@@ -313,7 +313,7 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.23.0 // indirect
 	goftp.io/server/v2 v2.0.0 // indirect
-	golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741 // indirect
+	golang.org/x/exp v0.0.0-20221006183845-316c7553db56 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20221005201717-2666ed6287c1 // indirect
 	golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
 	golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect

go.sum

@@ -542,6 +542,8 @@ github.com/hktalent/PipelineHttp v0.0.0-20221006102853-7270ca9cc3dc h1:dz5vNFzfG
 github.com/hktalent/PipelineHttp v0.0.0-20221006102853-7270ca9cc3dc/go.mod h1:ncw1+ugTc5GPQLUHHI7uWrgW2KWBppDBWwwjC984QJg=
 github.com/hktalent/PipelineHttp v0.0.0-20221006120713-f0d2d692285e h1:6Iy5XhPWznVIQEXJNFeE/RyXe3wrIfIjybW/yLAbch4=
 github.com/hktalent/PipelineHttp v0.0.0-20221006120713-f0d2d692285e/go.mod h1:ncw1+ugTc5GPQLUHHI7uWrgW2KWBppDBWwwjC984QJg=
+github.com/hktalent/PipelineHttp v0.0.0-20221007020748-952ec7255bdb h1:u639eutmwOJ1eBdyqvmCCq96v6tU1+b/PwIf1uaJvkE=
+github.com/hktalent/PipelineHttp v0.0.0-20221007020748-952ec7255bdb/go.mod h1:ncw1+ugTc5GPQLUHHI7uWrgW2KWBppDBWwwjC984QJg=
 github.com/hktalent/go-utils v0.0.0-20221004021941-7e10e0fb13ea h1:vuxZbB9vAwBi0Uj4F5GOfVtsi5E9MFX07EkCKypVu9M=
 github.com/hktalent/go-utils v0.0.0-20221004021941-7e10e0fb13ea/go.mod h1:9E0C0K+/zzyJ+VqFx1llC3y7+mGgW3toLoyMQnlNXhw=
 github.com/hktalent/go-utils v0.0.0-20221004095234-2e23f13b429d h1:z1IUP4hqn0LGgs78bU2gSlna92/p+RlB0MSZ+RxSmCo=
@@ -1279,6 +1281,8 @@ golang.org/x/exp v0.0.0-20221002003631-540bb7301a08 h1:LtBIgSqNhkuC9gA3BFjGy5obH
 golang.org/x/exp v0.0.0-20221002003631-540bb7301a08/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
 golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741 h1:fGZugkZk2UgYBxtpKmvub51Yno1LJDeEsRp2xGD+0gY=
 golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+golang.org/x/exp v0.0.0-20221006183845-316c7553db56 h1:BrYbdKcCNjLyrN6aKqXy4hPw9qGI8IATkj4EWv9Q+kQ=
+golang.org/x/exp v0.0.0-20221006183845-316c7553db56/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1483,6 +1487,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
 golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 h1:AzgQNqF+FKwyQ5LbVrVqOcuuFB67N47F9+htZYH0wFM=
+golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

lib/crawlergo/mychromedp.go

@@ -154,7 +154,7 @@ func (r *MyChromedp) DoUrlWithFlg(szUrl string, head *map[string]interface{}, ti
 	//if err := ioutil.WriteFile("screenshot1.png", b1, 0o644); err != nil {
 	//	log.Fatal(err)
 	//}
-	log.Println(title)
+	//log.Println(title)
 	//c1 := chromedp.FromContext(taskCtx)
 	return nil, cancel
 }

lib/util/asyncCmd.go

@@ -3,7 +3,6 @@ package util
 import (
 	"bufio"
 	"io"
-	"log"
 	"os"
 	"os/exec"
 	"time"
@@ -77,7 +76,7 @@ func (r *Cmd) WriteInput(args ...string) {
 
 func (r *Cmd) AsynCmd(fnCbk func(line string), szCmd string, args ...string) error {
 	cmd := r.Command(szCmd, args...)
-	log.Println(cmd.Args)
+	//log.Println(cmd.Args)
 	var err error
 	cmdReader, err := cmd.StdoutPipe()
 	if nil != err {

lib/util/config.go

@@ -113,7 +113,7 @@ var TmpFile = map[string][]*os.File{}
 func GetTempFile(t string) *os.File {
 	tempInput, err := ioutil.TempFile("", "scan4all-out*")
 	if err != nil {
-		log.Println(err)
+		//log.Println(err)
 		return nil
 	} else {
 		if t1, ok := TmpFile[t]; ok {
@@ -438,29 +438,29 @@ func TestIs404(szUrl string) (r01 *Response, err error, ok bool) {
 		}
 	}
 	sz404 := szUrl + Abs404
-	client := GetClient(sz404)
-	if nil != client {
-		//client.Client.Timeout = 500
-		//client.ErrCount = 0
-		//client.ErrLimit = 9999
-		//log.Printf("%v %s \n", client, sz404)
-	}
+	//client := GetClient(sz404)
+	//if nil != client {
+	//	client.Client.Timeout = 500
+	//	client.ErrCount = 0
+	//	//client.ErrLimit = 9999
+	//	//log.Printf("%v %s \n", client, sz404)
+	//}
 
 	//log.Println("start test ", sz404)
 	var mh1 map[string]string
-	if strings.HasPrefix(sz404, "http://") {
-		mh1 = map[string]string{
-			//"Connection":   "close",
-			"Content-Type": "",
-		}
-	}
+	//if strings.HasPrefix(sz404, "http://") {
+	//	mh1 = map[string]string{
+	//		//"Connection":   "close",
+	//		"Content-Type": "",
+	//	}
+	//}
 	r01, err = HttpRequset(sz404, "GET", "", false, mh1)
 	ok = err == nil && nil != r01 && 404 == r01.StatusCode
 	if nil != err {
 		CloseHttpClient(sz404)
 		//log.Println(sz404, err)
 	} else {
-		log.Printf("%d %s %s\n", r01.StatusCode, r01.Protocol, sz404)
+		//log.Printf("%d %s %s\n", r01.StatusCode, r01.Protocol, sz404)
 	}
 	noRpt.Set(key, []interface{}{r01, err, ok}, defaultInteractionDuration)
 	//client.Client.Timeout = 10

lib/util/db.go

@@ -72,7 +72,7 @@ func InitDb(dst ...interface{}) *gorm.DB {
 			log.Println("sqlite db init Connection failed", err)
 		}
 	} else {
-		log.Println(err)
+		//log.Println(err)
 	}
 	return dbCC
 }

lib/util/sv2es.go

@@ -105,7 +105,7 @@ func SendReq(data1 interface{}, id string, szType ESaveType) {
 				body, err := ioutil.ReadAll(resp.Body)
 				if nil == err && 0 < len(body) {
 					Log("Es save result ", string(body))
-				} else {
+				} else if nil != err {
 					Log(err)
 				}
 			}

main.go

@@ -23,6 +23,8 @@ var Version string
 func main() {
 	//os.Args = []string{"", "-host", "http://192.168.0.109", "-v"}
 	//os.Args = []string{"", "-host", "http://127.0.0.1", "-v"}
+	//os.Args = []string{"", "-list", "7b8fa7a85f9f6ae6f9178504d2202666fb8dc772.xml", "-v"}
+
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	util.DoInit(&config)
 	// set version

pkg/httpx/runner/options.go

@@ -273,7 +273,7 @@ func ParseOptions() *Options {
 	options.OutputMatchString = ""
 	options.OutputMatchRegex = ""
 	options.OutputExtractRegex = ""
-	options.OutputFilterStatusCode = ""
+	options.OutputFilterStatusCode = "400,404,500"
 	options.OutputFilterContentLength = ""
 	options.OutputFilterLinesCount = ""
 	options.OutputFilterWordsCount = ""

pkg/naabu/v2/pkg/runner/targets.go

@@ -190,12 +190,12 @@ func (r *Runner) DoTargets() (bool, error) {
 				s009 = "/config/doNmapScanWin.bat "
 			}
 			x := util.SzPwd + s009 + r.targetsFile + " " + tempInput1.Name()
-			log.Println(x)
+			//log.Println(x)
 			ss, err := util.DoCmd(strings.Split(x, " ")...)
 			s0 := tempInput1.Name()
 			if nil == err {
 				if "" != ss {
-					log.Println(ss, "\n")
+					//	log.Println(ss, "\n")
 				}
 				if util.FileExists(s0) {
 					//data, err := tempInput1.Stat()

pkg/portScan/masscan.go

@@ -79,7 +79,7 @@ func (m *Masscan) Run(fnCbk func(*models.Host)) error {
 	err := util.AsynCmd(func(line string) {
 		x1, err := m.ParseLine(line)
 		if nil != err {
-			log.Println(err)
+			//log.Println(err)
 			return
 		}
 		for _, i := range x1 {

pkg/portScan/nmapScan.go

@@ -105,7 +105,7 @@ func (s *Scanner) scan(nmapScanner *nmap.Scanner, fnCbk func(*Stream)) ([]*Strea
 	scanner1 := nmapScanner.GetStdout()
 	for scanner1.Scan() {
 		s091 := scanner1.Text()
-		log.Println(s091)
+		//log.Println(s091)
 		if r09, err := nmap.Parse([]byte(s091)); nil == err {
 			for _, host := range r09.Hosts {
 				if len(host.Ports) == 0 || len(host.Addresses) == 0 {
@@ -130,7 +130,7 @@ func (s *Scanner) scan(nmapScanner *nmap.Scanner, fnCbk func(*Stream)) ([]*Strea
 				}
 			}
 		} else {
-			log.Println(err)
+			//log.Println(err)
 		}
 	}
 	log.Printf("Found %d  Real Time Streaming Protocol (RTSP)\n", len(streams))

projectdiscovery/uncover/uncover.go

@@ -2,16 +2,39 @@ package uncover
 
 import (
 	"context"
+	"strings"
 
 	// Attempts to increase the OS file descriptors - Fail silently
 	_ "github.com/projectdiscovery/fdmax/autofdmax"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/uncover/runner"
 )
 
-func DoUncover() {
+// https://github.com/projectdiscovery/uncover
+/*
+Query multiple search engine at once
+Available Search engine support
+Shodan
+Censys
+FOFA
+Hunter
+Quake
+Zoomeye
+*/
+func DoUncover(targets []string) {
 	// Parse the command line flags and read config files
-	options := runner.ParseOptions()
+	options := &runner.Options{Provider: &runner.Provider{},
+		Query:   targets,
+		Engine:  strings.Split("shodan,shodan-idb,fofa,censys", ","),
+		Timeout: 30,
+		Delay:   1,
+		JSON:    true,
+		Limit:   10000,
+		NoColor: true,
+		Silent:  true,
+		Version: false,
+		Verbose: false,
+	}
 
 	newRunner, err := runner.NewRunner(options)
 	if err != nil {