添加poc 小工具tools/cve/main.go 2023-01-13

README.md

@@ -1,4 +1,4 @@
-[![Tweet](https://img.shields.io/twitter/szUrl/http/Hktalent3135773.svg?style=social)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![Follow on Twitter](https://img.shields.io/twitter/follow/Hktalent3135773.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![GitHub Followers](https://img.shields.io/github/followers/hktalent.svg?style=social&label=Follow)](https://github.com/hktalent/)
+[![Tweet](https://img.shields.io/twitter/url/http/Hktalent3135773.svg?style=social)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![Follow on Twitter](https://img.shields.io/twitter/follow/Hktalent3135773.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![GitHub Followers](https://img.shields.io/github/followers/hktalent.svg?style=social&label=Follow)](https://github.com/hktalent/)
 <p align="center">
    <a href="/README_CN.md">README_中文</a> •
    <a href="/static/Installation.md">Compile/Install/Run</a> •
@@ -64,6 +64,26 @@ noScan=true ./scan4all -l list.txt -v
 
 <img src="/static/nmap.gif" width="400">
 
+### --top-ports
+values for reaching various effectiveness levels
+https://nmap.org/book/performance-port-selection.html
+
+|Effectiveness|TCP portsrequired|UDP ports required|
+| ----------- | ----------- | ----------- |
+|10%|1|5|
+|20%|2|12|
+|30%|4|27|
+|40%|6|135|
+|50%|10|1,075|
+|60%|18|2,618|
+|70%|44|5,157|
+|80%|122|7,981|
+|85%|236|9,623|
+|90%|576|11,307|
+|95%|1,558|13,035|
+|99%|3,328|15,094|
+|100%|65,536|65,536|
+
 - Fast 15000+ POC detection capabilities, PoCs include: 
   * nuclei POC
   ## Nuclei Templates Top 10 statistics
@@ -129,7 +149,7 @@ mkdir ~/MyWork/;cd ~/MyWork/;git clone https://github.com/hktalent/log4j-scan
 download from
 <a href=https://github.com/hktalent/ProScan4all/releases>Releases</a>
 ```bash
-go install github.com/hktalent/[email protected]
+go install github.com/hktalent/scan4all@latest
 scan4all -h
 ````
 # how to use
@@ -191,14 +211,13 @@ more see: <a href=https://github.com/hktalent/ProScan4all/discussions>discussion
 - 2022-06-07 增加http url列表精准扫描参数，根据环境变量UrlPrecise=true开启
 
 # Communication group (WeChat, QQ，Tg)
-| Wechat | Or | QQchat | Or | Tg |
-| --- |--- |--- |--- |--- |
-
-|<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/wcq.JPG>||<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/qqc.jpg>||<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/tg.jpg>|
+| Wechat                                                                            | Or                                                                                | QQchat                                                                       | Or | Tg |
+|---|---|---|--- |--- |
+| <img width=166 src=https://github.com/hktalent/scan4all/blob/main/static/wcq.JPG> || <img width=166 src=https://github.com/hktalent/scan4all/blob/main/static/qqc.jpg> || <img width=166 src=https://github.com/hktalent/sall/blob/main/static/tg.jpg> |
 
 
 ## 💖Star
-[![Stargazers over time](https://starchart.cc/hktalent/ProScan4all.svg)](https://starchart.cc/hktalent/ProScan4all)
+[![Stargazers over time](https://starchart.cc/hktalent/scan4all.svg)](https://starchart.cc/hktalent/scan4all)
 
 # Donation
 | Wechat Pay | AliPay | Paypal | BTC Pay |BCH Pay |

README_CN.md

@@ -1,4 +1,4 @@
-[![Tweet](https://img.shields.io/twitter/szUrl/http/Hktalent3135773.svg?style=social)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![Follow on Twitter](https://img.shields.io/twitter/follow/Hktalent3135773.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![GitHub Followers](https://img.shields.io/github/followers/hktalent.svg?style=social&label=Follow)](https://github.com/hktalent/)
+[![Tweet](https://img.shields.io/twitter/url/http/Hktalent3135773.svg?style=social)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![Follow on Twitter](https://img.shields.io/twitter/follow/Hktalent3135773.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=Hktalent3135773) [![GitHub Followers](https://img.shields.io/github/followers/hktalent.svg?style=social&label=Follow)](https://github.com/hktalent/)
 <p align="center">
    <a href="/README.md">README_EN</a> •
    <a href="/static/Installation.md">编译/安装/运行</a> •
@@ -64,6 +64,25 @@ noScan=true  ./scan4all -l list.txt  -v
 
 <img src="/static/nmap.gif" width="400">
 
+### --top-ports
+values for reaching various effectiveness levels
+https://nmap.org/book/performance-port-selection.html
+|Effectiveness|TCP portsrequired|UDP ports required|
+| --- | --- | --- |
+|10%|1|5|
+|20%|2|12|
+|30%|4|27|
+|40%|6|135|
+|50%|10|1,075|
+|60%|18|2,618|
+|70%|44|5,157|
+|80%|122|7,981|
+|85%|236|9,623|
+|90%|576|11,307|
+|95%|1,558|13,035|
+|99%|3,328|15,094|
+|100%|65,536|65,536|
+
 - 快速 15000+ POC 检测功能，PoCs包含：
   * nuclei POC
   ## Nuclei Templates Top 10 statistics
@@ -130,7 +149,7 @@ mkdir ~/MyWork/;cd ~/MyWork/;git clone  https://github.com/hktalent/log4j-scan
 download from
 <a href=https://github.com/hktalent/ProScan4all/releases>Releases</a>
 ```bash
-go install github.com/hktalent/[email protected]
+go install github.com/hktalent/scan4all@latest
 scan4all -h
 ```
 # 如何使用
@@ -243,12 +262,12 @@ more see: <a href=https://github.com/hktalent/ProScan4all/discussions>discussion
 
 
 ## 💖Star
-[![Stargazers over time](https://starchart.cc/hktalent/ProScan4all.svg)](https://starchart.cc/hktalent/ProScan4all)
+[![Stargazers over time](https://starchart.cc/hktalent/scan4all.svg)](https://starchart.cc/hktalent/scan4all)
 
 # Donation
 | Wechat Pay | AliPay | Paypal | BTC Pay |BCH Pay |
 | --- | --- | --- | --- | --- |
-|<img src=https://github.com/hktalent/myhktools/blob/master/md/wc.png>|<img width=166 src=https://github.com/hktalent/myhktools/blob/master/md/zfb.png>|[paypal](https://www.paypal.me/pwned2019) **[email protected]**|<img width=166 src=https://github.com/hktalent/myhktools/blob/master/md/BTC.png>|<img width=166 src=https://github.com/hktalent/myhktools/blob/master/md/BCH.jpg>|
+|<img src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/wc.png>|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/zfb.png>|[paypal](https://www.paypal.me/pwned2019) **[email protected]**|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/BTC.png>|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/BCH.jpg>|
 
 
 <!--

brute/dicts/httpass.txt

@@ -1,6 +1,7 @@
 000000
 
 000000a
+minioadmin
 abcabc
 qwerty123
 100200

brute/dicts/httpuser.txt

@@ -14,4 +14,5 @@ root
 server_admin
 test
 tomcat
-xampp
+xampp
+minioadmin

brute/filefuzz.go

@@ -5,10 +5,9 @@ import (
 	_ "embed"
 	"fmt"
 	"github.com/antlabs/strsim"
-	"github.com/hktalent/51pwnPlatform/lib/scan/Const"
-	"github.com/hktalent/51pwnPlatform/pkg/models"
 	"github.com/hktalent/ProScan4all/lib/util"
 	"github.com/hktalent/ProScan4all/pkg/fingerprint"
+	Const "github.com/hktalent/go-utils"
 	"io/ioutil"
 	"log"
 	"mime"
@@ -157,9 +156,12 @@ func init() {
 		}
 		//regs = append(regs, ret...)
 		// 基于工厂方法构建
-		util.EngineFuncFactory(Const.ScanType_WebDirScan, func(evt *models.EventData, args ...interface{}) {
-			filePaths, fileFuzzTechnologies := FileFuzz(evt.Task.ScanWeb, 200, 100, "")
-			util.SendEngineLog(evt, Const.ScanType_WebDirScan, filePaths, fileFuzzTechnologies)
+		util.EngineFuncFactory(Const.ScanType_WebDirScan, func(evt *Const.EventData, args ...interface{}) {
+			for _, x := range evt.EventData {
+				szT := fmt.Sprintf("%v", x)
+				filePaths, fileFuzzTechnologies := FileFuzz(szT, 200, 100, "")
+				util.SendEngineLog(evt, Const.ScanType_WebDirScan, filePaths, fileFuzzTechnologies)
+			}
 		})
 
 		// 注册一个
@@ -185,8 +187,9 @@ var r001 = regexp.MustCompile(`\.(aac)|(abw)|(arc)|(avif)|(avi)|(azw)|(bin)|(bmp
 
 // 重写了fuzz：优化流程、优化算法、修复线程安全bug、增加智能功能
 //
-//	两次  ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
-//	去除指纹请求的路径，避免重复
+//		两次  ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
+//		去除指纹请求的路径，避免重复
+//	  当前域名ip <- x -> 域名，互相转换后，确认避免重复执行
 func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody string) ([]string, []string) {
 	DoInitMap()
 	u01, err := url.Parse(strings.TrimSpace(u))

config/config.json

@@ -156,8 +156,9 @@
     ],
     "naabu":[
       "-l", "",
-      "-top-ports", "http",
+      "-top-ports", "full",
       "-iv", "4,6",
+      "-exclude-cdn",
       "-scan-type", "s",
       "-sa","-silent","-nc",
       "-c", "64",

config/doNmapScan.sh

@@ -5,7 +5,7 @@ function doMasScan {
         # -F --top-ports=65535
         #  -p 80,443
         # -sV 得到的指纹信息更准，但是更慢
-        echo $PPSSWWDD|sudo -S nmap -F  --top-ports=65535 -n --unique --resolve-all -Pn -sU -sS --min-hostgroup 64 --max-retries 0 --host-timeout 10m --script-timeout 3m --version-intensity 9 --min-rate ${XRate} -T4  -iL $1 -oX $2
+        echo $PPSSWWDD|sudo -S nmap -F --top-ports=65535 -n --unique --resolve-all -Pn -sU -sS --min-hostgroup 64 --max-retries 0 --host-timeout 10m --script-timeout 3m --version-intensity 9 --min-rate ${XRate} -T4  -iL $1 -oX $2
     else
         echo $PPSSWWDD|sudo -S nmap -F --top-ports=65535 -n --unique --resolve-all -Pn -sU -sS --min-hostgroup 64 --max-retries 0 --host-timeout 10m --script-timeout 3m --version-intensity 9 --min-rate ${XRate} -T4  $1 -oX $2
     fi

config/nuclei/.templates-config.json

@@ -1 +1 @@
-{"nuclei-templates-directory":"/Users/51pwn/MyWork/scan4all_release/config/nuclei-templates","nuclei-version":"2.7.9","nuclei-latest-version":"","nuclei-templates-latest-version":""}
+{"nuclei-templates-directory":"/Users/51pwn/nuclei-templates","custom-s3-templates-directory":"/Users/51pwn/nuclei-templates/s3","custom-github-templates-directory":"/Users/51pwn/nuclei-templates/github","nuclei-version":"2.8.6","nuclei-latest-version":"","nuclei-templates-latest-version":""}

engine/engineImp.go

@@ -5,13 +5,12 @@ import (
 	"context"
 	"fmt"
 	"github.com/asaskevich/govalidator"
-	"github.com/hktalent/51pwnPlatform/lib"
-	"github.com/hktalent/51pwnPlatform/pkg/models"
 	"github.com/hktalent/ProScan4all/lib/util"
 	"github.com/hktalent/ProScan4all/pocs_go"
 	Const "github.com/hktalent/go-utils"
 	"github.com/hktalent/jaeles/cmd"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/karlseguin/ccache"
 	"github.com/panjf2000/ants/v2"
 	"github.com/projectdiscovery/iputil"
 	"github.com/remeh/sizedwaitgroup"
@@ -37,12 +36,14 @@ type Engine struct {
 	Wg           *sizedwaitgroup.SizedWaitGroup // Wg
 	Pool         int                            // 线程池
 	PoolFunc     *ants.PoolWithFunc             // 线程调用
-	EventData    chan *models.EventData         // 数据队列
+	EventData    chan *Const.EventData          // 数据队列
 	NodeId       string                         `json:"node_id"`    // 分布式引擎节点的id，除非系统更换，docker重制，否则始终一致
 	LimitTask    int                            `json:"limit_task"` // 当前节点任务并发数的限制
 	SyTask       int                            `json:"sy_task"`    // 剩余task
 	DtServer     string                         `json:"dt_server"`  // 获取任务、提交任务状态的server
 	caseScanFunc sync.Map
+	Lock         *sync.Mutex
+	mcc          *ccache.Cache // 内存缓存
 }
 
 var GEngine *Engine
@@ -60,18 +61,20 @@ func NewEngine(c *context.Context, pool int) *Engine {
 		return util.G_Engine.(*Engine)
 	}
 	x1 := &Engine{
+		mcc:       util.GetMemoryCache(100000, nil),
+		Lock:      &sync.Mutex{},
 		Context:   c,
 		Wg:        util.GetWg(util.GetValAsInt("WgThread", 64)),
 		Pool:      pool,
 		DtServer:  util.GetVal("DtServer"),
-		EventData: make(chan *models.EventData, pool),
+		EventData: make(chan *Const.EventData, pool),
 		LimitTask: util.GetValAsInt("LimitTask", 4),
 	}
 	x1.SyTask = x1.LimitTask // 初始化剩余任务等于最大任务数
 	x1.initNodeId()
 	p, err := ants.NewPoolWithFunc(pool, func(i interface{}) {
 		defer x1.Wg.Done()
-		x1.DoEvent(i.(*models.EventData))
+		x1.DoEvent(i.(*Const.EventData))
 	}, ants.WithPreAlloc(true))
 	if nil != err {
 		log.Println("ants.NewPoolWithFunc is error: ", err)
@@ -112,7 +115,7 @@ func (e *Engine) GetTask(okTaskIds string) {
 		"Content-Type": "application/json",
 	}, strings.NewReader(`{"Num":`+strconv.Itoa(e.SyTask)+`,"task_ids":"`+okTaskIds+`","node_id":"`+e.NodeId+`","task_num":`+strconv.Itoa(e.LimitTask)+`}`)); nil == err && nil != resp {
 		defer resp.Body.Close()
-		var n1 = models.EventData{}
+		var n1 = Const.EventData{}
 		var oTsk = map[string]interface{}{}
 		if data, err := ioutil.ReadAll(resp.Body); nil == err {
 			if err := json.Unmarshal(data, &oTsk); nil == err {
@@ -231,7 +234,7 @@ func (e *Engine) SendTask(s string) {
 			"Content-Type": "application/json",
 		}, bytes.NewReader(data)); nil == err && nil != resp {
 			defer resp.Body.Close()
-			var n1 = models.EventData{}
+			var n1 = Const.EventData{}
 			if data, err := ioutil.ReadAll(resp.Body); nil == err {
 				if err := json.Unmarshal(data, &n1); nil == err {
 					e.SendEvent(&n1, n1.EventType)
@@ -242,12 +245,12 @@ func (e *Engine) SendTask(s string) {
 }
 
 // 注册特定类型的事件处理
-func (e *Engine) EngineFuncFactory(nT int64, fnCbk util.EngineFuncType) {
+func (e *Engine) EngineFuncFactory(nT uint64, fnCbk util.EngineFuncType) {
 	e.RegCaseScanFunc(nT, fnCbk)
 }
 
 // 注册特定类型的事件处理
-func (e *Engine) RegCaseScanFunc(nType int64, fnCbk util.EngineFuncType) {
+func (e *Engine) RegCaseScanFunc(nType uint64, fnCbk util.EngineFuncType) {
 	e.caseScanFunc.Store(nType, fnCbk)
 }
 
@@ -308,41 +311,63 @@ func (e *Engine) EventType2Str(argsTypes ...uint64) string {
 }
 
 // 关联发送若干个事件
-func (e *Engine) SendEvent(evt *models.EventData, argsTypes ...int64) {
+func (e *Engine) SendEvent(evt *Const.EventData, argsTypes ...uint64) {
 	for _, i := range argsTypes {
-		var n1 = models.EventData{}
+		var n1 = Const.EventData{}
 		deepcopier.Copy(evt).To(&n1)
 		n1.EventType = i
 		e.EventData <- &n1
 	}
 }
 
+// 7天
+var ScanTargetNoRepeatCc = time.Minute * 60 * 24 * 7
+
 // 分派任务
-func (e *Engine) Dispather(ed *models.EventData) {
+//
+//	1-加锁，避免多个任务并发冲突
+//	2-获取参数做key + type，避免重复执行
+func (e *Engine) Dispather(ed *Const.EventData) {
+	e.Lock.Lock()
+	defer e.Lock.Unlock()
 	oR := e.GetCaseScanFunc()
+	bNo := true
 	oR.Range(func(k, v any) bool {
-		t1 := k.(int64)
+		t1 := k.(uint64)
 		if t1&ed.EventType == t1 {
-			v.(util.EngineFuncType)(ed, ed.EventData...)
+			bNo = false
+			log.Println("Dispather ", Const.GetTypeName(t1), ed.EventData)
+			if 0 == len(ed.EventData) || fmt.Sprintf("%v", ed.EventData[0]) == "" {
+				log.Println("No correct parameters ", Const.GetTypeName(t1))
+				return true
+			}
+			szKey := fmt.Sprintf("%s_%s", Const.GetTypeName(t1), util.GetSha1(ed.EventData))
+			if nil == e.mcc.Get(szKey) {
+				e.mcc.Set(szKey, "", ScanTargetNoRepeatCc)
+				v.(util.EngineFuncType)(ed, ed.EventData...)
+			}
 		}
 		return true
 	})
+	if bNo {
+		log.Println("not found event type")
+	}
 }
 
 // 执行事件代码 内部用
 //
 //	每个事件自己做防重处理
 //	每个事件异步执行
 //	每种事件类型可以独立控制并发数
-func (e *Engine) DoEvent(ed *models.EventData) {
+func (e *Engine) DoEvent(ed *Const.EventData) {
 	if nil != ed && nil != ed.EventData && 0 < len(ed.EventData) {
 		e.Dispather(ed)
 	}
 }
 
 func (x1 *Engine) Running() {
 	// 异步启动一个线程处理检测，避免
-	util.DefaultPool.Submit(func() {
+	util.DoSyncFunc(func() {
 		defer func() {
 			x1.Close()
 		}()
@@ -351,8 +376,8 @@ func (x1 *Engine) Running() {
 		//nMax := 120 // 等xxx秒都没有消息进入就退出
 		//nCnt := 0
 		// 每10秒获取一次任务
-		c1Task := time.NewTicker(5 * time.Second)
-		c2Task := time.NewTicker(15 * time.Second)
+		c1Task := time.NewTicker(5 * time.Second)  // 获取分布式任务
+		c2Task := time.NewTicker(15 * time.Second) // 延时清理
 		for {
 			select {
 			case <-util.Ctx_global.Done():
@@ -399,8 +424,6 @@ func (x1 *Engine) Running() {
 // 引擎总入口
 func init() {
 	//log.Println("engineImp.go run")
-	lib.GConfigServer.OnClient = true
-	lib.MyHub.FnClose()
 	util.RegInitFunc4Hd(func() {
 		// 下面的变量 不能移动到DoSyncFunc，否则全局变量将影响后续的init，导致无效的内存
 		NewEngine(&util.Ctx_global, util.GetValAsInt("ScanPoolSize", 5000))