up 2022-11-18

.gitattributes

@@ -0,0 +1,2 @@
+config/rockyou.txt filter=lfs diff=lfs merge=lfs -text
+main filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -7,6 +7,8 @@
 *.so
 db/sqlite3.db
 *.dylib
+sqlite3.db
+tools/macOS
 release
 .DbCache
 scan4all_linux

.gitmodules

@@ -3,4 +3,4 @@
   url = https://github.com/hktalent/nuclei-templates.git
 [submodule "config/jaeles-signatures"]
 	path = config/jaeles-signatures
-	url = https://github.com/hktalent/jaeles-signatures.git
+	url = https://github.com/jaeles-project/jaeles-signatures

config/config.json

@@ -3,9 +3,15 @@
     "OWASP": ["Amass"],
     "ffuf": ["ffuf"],
     "OJ": ["gobuster"],
+    "jaeles-project":["jaeles"],
     "projectdiscovery": ["naabu","nuclei","subfinder","httpx","katana","interactsh","uncover","dnsx","shuffledns","tlsx","asnmap"]
   },
   "cmds":{
+    "jaeles": [
+      "scan",
+      "-c",20,
+      "-o","{PWD}/logs"
+    ],
     "ffuf": [
       "-u","",
       "-w","{PWD}/brute/dicts/filedic.txt",
@@ -221,8 +227,8 @@
   "nmap": "nmap -n --unique --resolve-all -Pn --min-hostgroup 64 --max-retries 0 --host-timeout 10m --script-timeout 3m -oX {filename} --version-intensity 9 --min-rate 10000 -T4 ",
   "HydraUser": "",
   "HydraPass": "",
-  "UrlPrecise": true,
-  "ParseSSl": true,
+  "UrlPrecise": false,
+  "ParseSSl": false,
   "EnableSubfinder": true,
   "EnableHoneyportDetection": true,
   "EnableKsubdomain": true,
@@ -261,8 +267,8 @@
     "MaxRedirects": 3
   },
   "enableEsSv": true,
-  "enableJaeles": false,
   "CheckWeakPassword": true,
+  "WgThread": 64,
   "jaelesThread": 8,
   "esthread": 8,
   "LimitTask": 4,

engine/engineImp.go

@@ -13,6 +13,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/panjf2000/ants/v2"
 	"github.com/projectdiscovery/iputil"
+	"github.com/remeh/sizedwaitgroup"
 	"github.com/ulule/deepcopier"
 	"io/ioutil"
 	"log"
@@ -31,15 +32,15 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary
 
 // 引擎对象，全局单实例
 type Engine struct {
-	Context      *context.Context       // 上下文
-	Wg           *sync.WaitGroup        // Wg
-	Pool         int                    // 线程池
-	PoolFunc     *ants.PoolWithFunc     // 线程调用
-	EventData    chan *models.EventData // 数据队列
-	NodeId       string                 `json:"node_id"`    // 分布式引擎节点的id，除非系统更换，docker重制，否则始终一致
-	LimitTask    int                    `json:"limit_task"` // 当前节点任务并发数的限制
-	SyTask       int                    `json:"sy_task"`    // 剩余task
-	DtServer     string                 `json:"dt_server"`  // 获取任务、提交任务状态的server
+	Context      *context.Context               // 上下文
+	Wg           *sizedwaitgroup.SizedWaitGroup // Wg
+	Pool         int                            // 线程池
+	PoolFunc     *ants.PoolWithFunc             // 线程调用
+	EventData    chan *models.EventData         // 数据队列
+	NodeId       string                         `json:"node_id"`    // 分布式引擎节点的id，除非系统更换，docker重制，否则始终一致
+	LimitTask    int                            `json:"limit_task"` // 当前节点任务并发数的限制
+	SyTask       int                            `json:"sy_task"`    // 剩余task
+	DtServer     string                         `json:"dt_server"`  // 获取任务、提交任务状态的server
 	caseScanFunc sync.Map
 }
 
@@ -56,10 +57,9 @@ func NewEngine(c *context.Context, pool int) *Engine {
 	if nil != util.G_Engine {
 		return util.G_Engine.(*Engine)
 	}
-
 	x1 := &Engine{
 		Context:   c,
-		Wg:        &sync.WaitGroup{},
+		Wg:        util.GetWg(util.GetValAsInt("WgThread", 64)),
 		Pool:      pool,
 		DtServer:  util.GetVal("DtServer"),
 		EventData: make(chan *models.EventData, pool),
@@ -113,7 +113,6 @@ func (e *Engine) GetTask(okTaskIds string) {
 		var oTsk = map[string]interface{}{}
 		if data, err := ioutil.ReadAll(resp.Body); nil == err {
 			if err := json.Unmarshal(data, &oTsk); nil == err {
-
 				e.SendEvent(&n1, n1.EventType)
 			}
 		}
@@ -238,11 +237,11 @@ func (e *Engine) SendTask(s string) {
 	}
 }
 
-func (e *Engine) EngineFuncFactory(nT int64, fnCbk interface{}) {
+func (e *Engine) EngineFuncFactory(nT int64, fnCbk util.EngineFuncType) {
 	e.RegCaseScanFunc(nT, fnCbk)
 }
 
-func (e *Engine) RegCaseScanFunc(nType int64, fnCbk interface{}) {
+func (e *Engine) RegCaseScanFunc(nType int64, fnCbk util.EngineFuncType) {
 	e.caseScanFunc.Store(nType, fnCbk)
 }
 
@@ -261,7 +260,11 @@ func (e *Engine) Close() {
 // case 扫描使用的函数
 func (e *Engine) DoCase(ed *models.EventData) util.EngineFuncType {
 	if i, ok := e.caseScanFunc.Load(ed.EventType); ok {
-		return i.(util.EngineFuncType)
+		if x, ok := i.(util.EngineFuncType); ok {
+			return x
+		} else {
+			log.Println(i)
+		}
 	}
 	return nil
 }
@@ -318,7 +321,7 @@ func (x1 *Engine) Running() {
 				}
 			case x2 := <-x1.EventData: // 各种扫描的控制
 				if nil != x2 && nil != x2.EventData {
-					x1.Wg.Add(1)
+					x1.Wg.Add()
 					x1.PoolFunc.Invoke(x2)
 				}
 			case x1, ok := <-util.PocCheck_pipe:

lib/util/Const.go

@@ -3,6 +3,7 @@ package util
 import (
 	"context"
 	"fmt"
+	util "github.com/hktalent/go-utils"
 	"github.com/remeh/sizedwaitgroup"
 	"net/http"
 	"os"
@@ -11,7 +12,13 @@ import (
 )
 
 // 全局线程控制
-var Wg = sizedwaitgroup.New(32)
+var Wg *sizedwaitgroup.SizedWaitGroup
+
+func init() {
+	RegInitFunc4Hd(func() {
+		Wg = GetWg(util.GetValAsInt("WgThread", 64))
+	})
+}
 
 // 全局控制
 var RootContext = context.Background()
@@ -25,6 +32,11 @@ var DeleteMe = regexp.MustCompile("rememberMe=deleteMe")
 // 自定义http 头
 var CustomHeaders []string
 
+func GetWg(n int) *sizedwaitgroup.SizedWaitGroup {
+	x1 := sizedwaitgroup.New(n)
+	return &x1
+}
+
 /*
 X-Forwarded-Host: 127.0.0.1
 X-Forwarded-For: 127.0.0.1

lib/util/delayClear.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"github.com/remeh/sizedwaitgroup"
 	"sync"
 	"time"
 )
@@ -64,15 +65,15 @@ func DoSleep() {
 }
 
 // 延时清理
-func DoDelayClear(Wg1 ...*sync.WaitGroup) {
-	var wg2 *sync.WaitGroup
+func DoDelayClear(Wg1 ...*sizedwaitgroup.SizedWaitGroup) {
+	var wg2 *sizedwaitgroup.SizedWaitGroup
 	if 0 < len(Wg1) && nil != Wg1[0] {
 		wg2 = Wg1[0]
 	} else {
 		wg2 = Wg
 	}
 	IsDo <- struct{}{}
-	wg2.Add(1)
+	wg2.Add()
 	go func() {
 		defer func() {
 			<-IsDo

lib/util/sv2es.go

@@ -100,14 +100,15 @@ func SendReq(data1 interface{}, id string, szType ESaveType) {
 	c1.ErrLimit = 10000
 	c1.ErrCount = 0
 	data, _ := json.Marshal(data1)
+	c1.UseHttp2 = true
 	c1.DoGetWithClient4SetHd(c1.GetClient4Http2(), szUrl, "POST", bytes.NewReader(data), func(resp *http.Response, err error, szU string) {
 		if nil != err {
 			log.Println("pphLog.DoGetWithClient4SetHd ", err)
 		} else {
 			defer resp.Body.Close()
 			body, err := ioutil.ReadAll(resp.Body)
 			if nil == err && 0 < len(body) {
-				Log("Es save result ", string(body))
+				Log("Es save result ", resp.StatusCode, string(body))
 			} else if nil != err {
 				Log(err)
 			}

lib/util/util.go

@@ -459,7 +459,7 @@ type EngineFuncType func(evt *models.EventData, args ...interface{})
 
 // 工厂方法
 //   便于同一、规范引擎调用的方法、参数约束
-var EngineFuncFactory func(nT int64, fnCbk interface{})
+var EngineFuncFactory func(nT int64, fnCbk EngineFuncType)
 
 // 全局引擎
 var G_Engine interface{}

main.go

@@ -24,7 +24,7 @@ var config embed.FS
 var Version string
 
 func main() {
-	os.Args = []string{"", "-host", "http://192.168.0.109", "-v"}
+	os.Args = []string{"", "-host", "http://127.0.0.1", "-v"}
 	//os.Args = []string{"", "-host", "http://127.0.0.1", "-v"}
 	//os.Args = []string{"", "-host", "https://www.sina.com.cn/", "-v", "-o", "xxx.csv"}
 	//os.Args = []string{"", "-list", "list.txt", "-v"}

pkg/naabu/v2/pkg/runner/targets.go

@@ -112,7 +112,7 @@ func (r *Runner) MergeToFile() (string, error) {
 
 func (r *Runner) DoSsl(target string) []string {
 	// 处理ssl 数字证书中包含的域名信息，深度挖掘漏洞
-	if "true" == util.GetVal("ParseSSl") {
+	if util.GetValAsBool("ParseSSl") {
 		aH, err := pkg.DoDns(target)
 		if nil == err {
 			return aH
@@ -339,8 +339,7 @@ func (r *Runner) AddTarget(target string) error {
 				//Add2Naabubuffer(u.Hostname())
 				// target 长度 大于 s1才处理
 				////UrlPrecise     bool // 精准url扫描，不去除url清单上下文 2022-06-08
-				UrlPrecise := util.GetVal(util.UrlPrecise)
-				if "true" == UrlPrecise && len(target) > len(s1) {
+				if util.GetValAsBool(util.UrlPrecise) && len(target) > len(s1) {
 					s2 := r1.ReplaceAllString(target[len(s1):], "")
 					// 包含1个以上/表示有上下文
 					if 1 < len(s2) {

pkg/xcmd/allCmdTools.go

@@ -112,3 +112,10 @@ func DoRaw4FuzzCmd(s, t string) string {
 	szName, _ := GetTempFile()
 	return doTpCmd(t, s, szName)
 }
+
+// 执行nmap
+func doNmap(s string) string {
+	s = Target2HostsFile(s)
+	szName, _ := GetTempFile()
+	return ""
+}

pkg/xcmd/utils.go

@@ -26,7 +26,7 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 	util.RegInitFunc(func() {
 		Pwd, _ = os.Getwd()
-		ToolsPath = Pwd + "/../tools/"
+		ToolsPath = Pwd + "/tools/"
 		envParm["PWD"] = Pwd
 	})
 }

projectdiscovery/nuclei_Yaml/nuclei_yaml.go

@@ -7,7 +7,6 @@ import (
 	"github.com/hktalent/51pwnPlatform/pkg/models"
 	"github.com/hktalent/ProScan4all/lib/util"
 	runner2 "github.com/hktalent/ProScan4all/projectdiscovery/nuclei_Yaml/nclruner/runner"
-	"github.com/hktalent/ProScan4all/webScan/jaeles"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/goflags"
@@ -147,9 +146,6 @@ func RunNuclei(buf *bytes.Buffer, xx chan bool, oOpts *map[string]interface{}, o
 
 	// 启动web扫描
 	util.Wg.Add()
-	if util.GetValAsBool("enableJaeles") {
-		go jaeles.RunScan(a66, "")
-	}
 
 	options.Targets = *x55
 	log.Printf("nuclei options.Targets = %+v\n", options.Targets)

simple/naabu.json

@@ -0,0 +1,4 @@
+{"ip":"192.168.0.109","port":8001,"timestamp":"2022-11-18T03:57:01.843856Z"}
+{"ip":"192.168.0.109","port":3240,"timestamp":"2022-11-18T03:57:01.843856Z"}
+{"ip":"192.168.0.109","port":548,"timestamp":"2022-11-18T03:57:01.843856Z"}
+{"ip":"192.168.0.109","port":111,"timestamp":"2022-11-18T03:57:01.843856Z"}