Merge branch 'main' into bbot-integration

CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## [0.6.0](https://github.com/freelabz/secator/compare/v0.5.2...v0.6.0) (2024-07-25)
+
+
+### Features
+
+* add duplicate finder to mongodb hooks ([#409](https://github.com/freelabz/secator/issues/409)) ([fb0e11c](https://github.com/freelabz/secator/commit/fb0e11cd2b64bf51bc862f47243c8c0602d3d5e9))
+* basic helm chart ([#408](https://github.com/freelabz/secator/issues/408)) ([6b2f84f](https://github.com/freelabz/secator/commit/6b2f84f61bd8eccf2cdd61b6ffdc2eb4489240bc))
+
+
+### Bug Fixes
+
+* Dockerfile broken apt install ([#407](https://github.com/freelabz/secator/issues/407)) ([c023279](https://github.com/freelabz/secator/commit/c02327968ecea816004636801684b336735df439))
+* **tasks:** duplicate meta opt entry ([#401](https://github.com/freelabz/secator/issues/401)) ([ae56aa6](https://github.com/freelabz/secator/commit/ae56aa62f5a18936a1787547e37bbe636e6e43c3))
+
+## [0.5.2](https://github.com/freelabz/secator/compare/v0.5.1...v0.5.2) (2024-05-07)
+
+
+### Bug Fixes
+
+* **nuclei,katana:** add -sr flag and write http responses and screenshot to correct folder ([#395](https://github.com/freelabz/secator/issues/395)) ([1a51790](https://github.com/freelabz/secator/commit/1a51790c9231f593631c2780b6d5e0fa89f1aa55))
+
+## [0.5.1](https://github.com/freelabz/secator/compare/v0.5.0...v0.5.1) (2024-05-06)
+
+
+### Bug Fixes
+
+* **output:** add headers to Url and print HTTP method when not GET ([#390](https://github.com/freelabz/secator/issues/390)) ([5a87d7b](https://github.com/freelabz/secator/commit/5a87d7b8bc1dd098999f3864952e98068fd32efc))
+* **report:** do not remove duplicate in reports by default ([#392](https://github.com/freelabz/secator/issues/392)) ([7d74ae8](https://github.com/freelabz/secator/commit/7d74ae80bfd99c31714a5e7e25f2bd1caa642eb4))
+
+## [0.5.0](https://github.com/freelabz/secator/compare/v0.4.1...v0.5.0) (2024-05-03)
+
+
+### Features
+
+* add searchsploit output fields ([#278](https://github.com/freelabz/secator/issues/278)) ([00872c4](https://github.com/freelabz/secator/commit/00872c4a7f9b1ec76ee1bfd7a00919d53cbdb30a))
+* **cli:** add report list / export commands ([#367](https://github.com/freelabz/secator/issues/367)) ([ab396a3](https://github.com/freelabz/secator/commit/ab396a3098c6d4c46cf9c9b29bd5c54579421646))
+* **config:** load external tasks from template dir ([#373](https://github.com/freelabz/secator/issues/373)) ([0c63c02](https://github.com/freelabz/secator/commit/0c63c02c8eca477a6752f4af466c4303801019de))
+
+
+### Bug Fixes
+
+* **cli:** catch JSON parse errors ([#378](https://github.com/freelabz/secator/issues/378)) ([5e3d7f2](https://github.com/freelabz/secator/commit/5e3d7f2d2938a857e7599a429a6cfabf3b12347b))
+* **nmap:** resolve -sS tcp syn stealth issue ([#376](https://github.com/freelabz/secator/issues/376)) ([a3efc65](https://github.com/freelabz/secator/commit/a3efc651dfa4d8fa34d611b9aea2e156352fdc45))
+
 ## [0.4.1](https://github.com/freelabz/secator/compare/v0.4.0...v0.4.1) (2024-04-30)
 
 

Dockerfile

@@ -22,8 +22,7 @@ RUN apt update -y && \
     jq \
     openssl \
 	proxychains \
-	proxychains-ng \
-	&& rm -rf /var/lib/apt/lists/*
+	proxychains-ng
 
 # Install Metasploit framework
 RUN curl https://raw.githubusercontent.com/rapid7/metasploit-omnibus/master/config/templates/metasploit-framework-wrappers/msfupdate.erb > msfinstall
@@ -43,5 +42,8 @@ RUN secator install addons mongodb
 RUN secator install addons redis
 RUN secator install addons dev
 
+# Cleanup
+RUN rm -rf /var/lib/apt/lists/*
+
 # Set entrypoint
 ENTRYPOINT ["secator"]

helm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: Secator
+description: A Secator Helm chart for Kubernetes
+
+type: application
+version: 0.1.0
+appVersion: "0.5.2"

helm/templates/redis-service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.redis.name }}
+  namespace: {{ .Values.namespace }}
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.redis.port }}
+    name: client
+  selector:
+    app: redis

helm/templates/redis.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Values.redis.name }}
+  namespace: {{ .Values.namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: redis
+  serviceName: {{ .Values.redis.name }}
+  replicas: {{ .Values.redis.replicas }}
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: {{ .Values.redis.name }}
+        image: {{ .Values.redis.image }}
+        ports:
+        - containerPort: {{ .Values.redis.port }}
+          name: client

helm/templates/secator-manager.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ .Values.secatorManager.name }}
+  namespace: {{ .Values.namespace }}
+  labels:
+    name: secator-manager
+spec:
+  containers:
+  - name: secator-manager
+    image: {{ .Values.secatorManager.image }}
+    command: ["tail"]
+    args: ["-F", "anything"]
+    env:
+    - name: SECATOR_CELERY_BROKER_URL
+      value: "redis://{{ .Values.redis.name }}:6379/0"
+    - name: SECATOR_CELERY_RESULT_BACKEND
+      value: "redis://{{ .Values.redis.name }}:6379/0"

helm/templates/secator-worker.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.secatorWorker.name }}
+  namespace: {{ .Values.namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: secator-worker
+  template:
+    metadata:
+      labels:
+        app: secator-worker
+    spec:
+      containers:
+      - name: {{ .Values.secatorWorker.name }}
+        image: {{ .Values.secatorWorker.image }}
+        command: ["secator"]
+        args: ["worker"]
+        env:
+        - name: SECATOR_CELERY_BROKER_URL
+          value: "redis://{{ .Values.redis.name }}:6379/0"
+        - name: SECATOR_CELERY_RESULT_BACKEND
+          value: "redis://{{ .Values.redis.name }}:6379/0"

helm/values.yaml

@@ -0,0 +1,34 @@
+# Default values for Secator
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+namespace: secator
+
+secatorManager:
+  name: secator-manager
+  image: "freelabz/secator"
+
+  # Empty if using default repository
+  repository:
+  # Empty if using tag "latest"
+  tag:
+
+secatorWorker:
+  name: secator-worker
+  image: "freelabz/secator"
+
+  # Empty if using default repository
+  repository:
+  # Empty if using tag "latest"
+  tag:
+
+redis:
+  name: redis
+  image: "redis"
+
+  # Empty if using default repository
+  repository:
+  # Empty if using tag "latest"
+  tag:
+  replicas: 1
+  port: 6379

package.json

@@ -0,0 +1,29 @@
+{
+	"name": "secator-worker",
+	"version": "0.0.1",
+	"description": "Secator worker",
+	"main": "",
+	"scripts": {
+		"solo": "npm run venv && venv/bin/secator worker -r",
+		"dev": "npm run venv && SECATOR_CELERY_BROKER_URL=redis://localhost:6379 SECATOR_CELERY_RESULT_BACKEND=redis://localhost:6379 venv/bin/secator worker -r",
+		"venv": "pip install virtualenv --break-system-packages && virtualenv venv && chmod +x venv/bin/activate && . venv/bin/activate && venv/bin/pip install -e .[dev,worker,redis,mongodb,trace]",
+		"generate": "rm -r venv && npm run venv && venv/bin/pip install fastapi uvicorn && venv/bin/pip freeze > requirements.txt",
+		"docker:build": "docker build -t secator .",
+		"docker:push": "gcloud builds submit .",
+		"docker:logs": "docker logs -f secator",
+		"docker:start": "docker run --name=secator-worker --network=host -e SECATOR_CELERY_BROKER_URL=redis://localhost:6379 -e SECATOR_CELERY_RESULT_BACKEND=redis://localhost:6379 -d secator worker",
+		"docker:stop": "docker stop secator-worker; docker rm secator-worker",
+		"docker:start-redis": "docker run --name redis -p 6379:6379 -d redis",
+		"docker:stop-redis": "docker stop redis; docker rm redis",
+		"docker:start-mongodb": "docker run --name mongodb -p 27017:27017 -d mongo:latest",
+		"docker:stop-mongodb": "docker stop mongodb; docker rm mongodb",
+		"docker:start-deps": "npm run docker:start-mongodb; npm run docker:start-redis",
+		"docker:stop-deps": "npm run docker:stop-mongodb; npm run docker:stop-redis",
+		"docker:start-all": "npm run docker:start-deps; npm run docker:start",
+		"docker:stop-all": "npm run docker:stop-deps; npm run docker:stop"
+	},
+	"keywords": [],
+	"author": "[email protected]",
+	"license": "",
+	"dependencies": {}
+}

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "secator"
-version = "0.4.1"
+version = "0.6.0"
 authors = [{ name = "FreeLabz", email = "[email protected]" }]
 readme = "README.md"
 description = "The pentester's swiss knife."
@@ -30,7 +30,7 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
-  "bs4 < 1",
+  "beautifulsoup4 <= 5",
   'celery < 6',
   "cpe < 2",
   "dotmap < 2",
@@ -46,6 +46,7 @@ dependencies = [
   "requests < 3",
   "rich < 14",
   "rich-click < 1.7",
+  "tldextract < 6",
   "typing_extensions < 5",
   "validators < 1",
   "xmltodict < 1"

secator/celery.py

@@ -1,11 +1,10 @@
 import gc
 import logging
 import traceback
-from time import sleep
 
 from celery import Celery, chain, chord, signals
 from celery.app import trace
-from celery.result import AsyncResult, allow_join_result
+from celery.result import allow_join_result
 # from pyinstrument import Profiler  # TODO: make pyinstrument optional
 from rich.logging import RichHandler
 
@@ -146,8 +145,7 @@ def break_task(task_cls, task_opts, targets, results=[], chunk_size=1):
 
 @app.task(bind=True)
 def run_task(self, args=[], kwargs={}):
-	if CONFIG.debug.level > 1:
-		logger.info(f'Received task with args {args} and kwargs {kwargs}')
+	debug(f'Received task with args {args} and kwargs {kwargs}', sub="celery", level=2)
 	if 'context' not in kwargs:
 		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
@@ -157,8 +155,7 @@ def run_task(self, args=[], kwargs={}):
 
 @app.task(bind=True)
 def run_workflow(self, args=[], kwargs={}):
-	if CONFIG.debug.level > 1:
-		logger.info(f'Received workflow with args {args} and kwargs {kwargs}')
+	debug(f'Received workflow with args {args} and kwargs {kwargs}', sub="celery", level=2)
 	if 'context' not in kwargs:
 		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
@@ -168,8 +165,7 @@ def run_workflow(self, args=[], kwargs={}):
 
 @app.task(bind=True)
 def run_scan(self, args=[], kwargs={}):
-	if CONFIG.debug.level > 1:
-		logger.info(f'Received scan with args {args} and kwargs {kwargs}')
+	debug(f'Received scan with args {args} and kwargs {kwargs}', sub="celery", level=2)
 	if 'context' not in kwargs:
 		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
@@ -354,56 +350,9 @@ def forward_results(results):
 	results = deduplicate(results, attr='_uuid')
 	return results
 
-
-#---------------------#
-# Celery result utils #
-#---------------------#
-
-
-def poll_task(result, seen=[]):
-	"""Poll Celery result tree recursively to get results live.
-
-	TODO: function is incomplete, as it does not parse all results.
-
-	Args:
-		result (Union[AsyncResult, GroupResult]): Celery result object.
-		seen (list): List of seen results (do not yield again).
-
-	Yields:
-		dict: Result.
-	"""
-	if result is None:
-		return
-
-	if result.children:
-		for child in result.children:
-			yield from poll_task(child, seen=seen)
-	else:
-		res = AsyncResult(result.id)
-		if not res.info:
-			sleep(0.1)
-			yield from poll_task(result, seen=seen)
-
-		# Task done running
-		if isinstance(res.info, list):
-			for item in res.info:
-				if item._uuid not in seen:
-					yield res.id, None, item
-					seen.append(item._uuid)
-			return
-
-		# Get task partial results, remove duplicates
-		results = res.info['results']
-		name = res.info['name']
-		for item in results:
-			if item._uuid not in seen:
-				yield res.id, name, item
-				seen.append(item._uuid)
-
-		# Task still running, keep polling
-		if not res.ready():
-			sleep(0.1)
-			yield from poll_task(result, seen=seen)
+#--------------#
+# Celery utils #
+#--------------#
 
 
 def is_celery_worker_alive():