Merge branch 'main' into httpx-tls-grab

package.json

@@ -6,7 +6,7 @@
 	"scripts": {
 		"solo": "npm run venv && venv/bin/secator worker -r",
 		"dev": "npm run venv && SECATOR_CELERY_BROKER_URL=redis://localhost:6379 SECATOR_CELERY_RESULT_BACKEND=redis://localhost:6379 venv/bin/secator worker -r",
-		"venv": "pip install virtualenv --break-system-packages && virtualenv venv && chmod +x venv/bin/activate && . venv/bin/activate && venv/bin/pip install -e .[dev,worker,redis,mongodb,trace,dev]",
+		"venv": "pip install virtualenv --break-system-packages && virtualenv venv && chmod +x venv/bin/activate && . venv/bin/activate && venv/bin/pip install -e .[dev,worker,redis,mongodb,trace]",
 		"generate": "rm -r venv && npm run venv && venv/bin/pip install fastapi uvicorn && venv/bin/pip freeze > requirements.txt",
 		"docker:build": "docker build -t secator .",
 		"docker:push": "gcloud builds submit .",

secator/celery.py

@@ -145,8 +145,7 @@ def break_task(task_cls, task_opts, targets, results=[], chunk_size=1):
 
 @app.task(bind=True)
 def run_task(self, args=[], kwargs={}):
-	if CONFIG.debug.level > 1:
-		logger.info(f'Received task with args {args} and kwargs {kwargs}')
+	debug(f'Received task with args {args} and kwargs {kwargs}', sub="celery", level=2)
 	if 'context' not in kwargs:
 		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
@@ -156,8 +155,7 @@ def run_task(self, args=[], kwargs={}):
 
 @app.task(bind=True)
 def run_workflow(self, args=[], kwargs={}):
-	if CONFIG.debug.level > 1:
-		logger.info(f'Received workflow with args {args} and kwargs {kwargs}')
+	debug(f'Received workflow with args {args} and kwargs {kwargs}', sub="celery", level=2)
 	if 'context' not in kwargs:
 		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
@@ -167,8 +165,7 @@ def run_workflow(self, args=[], kwargs={}):
 
 @app.task(bind=True)
 def run_scan(self, args=[], kwargs={}):
-	if CONFIG.debug.level > 1:
-		logger.info(f'Received scan with args {args} and kwargs {kwargs}')
+	debug(f'Received scan with args {args} and kwargs {kwargs}', sub="celery", level=2)
 	if 'context' not in kwargs:
 		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id

secator/configs/workflows/host_recon.yaml

@@ -9,8 +9,11 @@ input_types:
 tasks:
   naabu:
     description: Find open ports
+    ports: "-"  # scan all ports
   nmap:
     description: Search for vulnerabilities on open ports
+    skip_host_discovery: True
+    version_detection: True
     targets_: port.host
     ports_: port.port
   httpx:
@@ -38,4 +41,4 @@ results:
   #   condition: item.confidence == 'high'
 
   - type: url
-    condition: item.status_code != 0
+    condition: item.status_code != 0

secator/configs/workflows/port_scan.yaml

@@ -5,11 +5,15 @@ description: Port scan
 tags: [recon, network, http, vuln]
 input_types:
   - host
+  - cidr_range
 tasks:
   naabu:
     description: Find open ports
+    ports: "-"  # scan all ports
   nmap:
     description: Search for vulnerabilities on open ports
+    skip_host_discovery: True
+    version_detection: True
     targets_: port.host
     ports_: port.port
   _group:
@@ -31,4 +35,4 @@ results:
   - type: url
     condition: item.status_code != 0
 
-  - type: vulnerability
+  - type: vulnerability

secator/decorators.py

@@ -269,9 +269,17 @@ def func(ctx, **opts):
 		driver = opts.pop('driver', '')
 		show = opts['show']
 		context = {'workspace_name': ws}
+
+		# Remove options whose values are default values
+		for k, v in options.items():
+			opt_name = k.replace('-', '_')
+			if opt_name in opts and opts[opt_name] == v.get('default', None):
+				del opts[opt_name]
+
 		# TODO: maybe allow this in the future
 		# unknown_opts = get_unknown_opts(ctx)
 		# opts.update(unknown_opts)
+
 		targets = opts.pop(input_type)
 		targets = expand_input(targets)
 		if sync or show:

secator/definitions.py

@@ -54,6 +54,7 @@
 HEADER = 'header'
 HOST = 'host'
 IP = 'ip'
+PROTOCOL = 'protocol'
 LINES = 'lines'
 METHOD = 'method'
 MATCH_CODES = 'match_codes'

secator/output_types/port.py

@@ -14,6 +14,7 @@ class Port(OutputType):
 	service_name: str = field(default='', compare=False)
 	cpes: list = field(default_factory=list, compare=False)
 	host: str = field(default='', repr=True, compare=False)
+	protocol: str = field(default='TCP', repr=True, compare=False)
 	extra_data: dict = field(default_factory=dict, compare=False)
 	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
 	_source: str = field(default='', repr=True, compare=False)
@@ -38,6 +39,8 @@ def __str__(self) -> str:
 
 	def __repr__(self) -> str:
 		s = f'🔓 {self.ip}:[bold red]{self.port:<4}[/] [bold yellow]{self.state.upper()}[/]'
+		if self.protocol != 'TCP':
+			s += f' \[[yellow3]{self.protocol}[/]]'
 		if self.service_name:
 			s += f' \[[bold purple]{self.service_name}[/]]'
 		if self.host:

secator/report.py

@@ -49,6 +49,7 @@ def build(self):
 				'title': self.title,
 				'runner': self.runner.__class__.__name__,
 				'name': self.runner.config.name,
+				'status': self.runner.status,
 				'targets': self.runner.targets,
 				'total_time': str(self.runner.elapsed),
 				'total_human': self.runner.elapsed_human,

secator/runners/_base.py

@@ -29,14 +29,11 @@
 HOOKS = [
 	'before_init',
 	'on_init',
-	'on_start',
 	'on_end',
 	'on_item_pre_convert',
 	'on_item',
 	'on_duplicate',
-	'on_line',
 	'on_iter',
-	'on_error',
 ]
 
 VALIDATORS = [
@@ -158,10 +155,10 @@ def __init__(self, config, targets, results=[], run_opts={}, hooks={}, context={
 
 		# Hooks
 		self.raise_on_error = self.run_opts.get('raise_on_error', False)
-		self.hooks = {name: [] for name in HOOKS}
+		self.hooks = {name: [] for name in HOOKS + getattr(self, 'hooks', [])}
 		for key in self.hooks:
 
-			# Register class specific hooks
+			# Register class + derived class hooks
 			class_hook = getattr(self, key, None)
 			if class_hook:
 				name = f'{self.__class__.__name__}.{key}'

secator/runners/command.py

@@ -82,6 +82,14 @@ class Command(Runner):
 	item_loader = None
 	item_loaders = [JSONSerializer(),]
 
+	# Hooks
+	hooks = [
+		'on_start',
+		'on_cmd',
+		'on_line',
+		'on_error',
+	]
+
 	# Ignore return code
 	ignore_return_code = False
 
@@ -141,6 +149,9 @@ def __init__(self, input=None, **run_opts):
 		# Build command
 		self._build_cmd()
 
+		# Run on_cmd hook
+		self.run_hooks('on_cmd')
+
 		# Build item loaders
 		instance_func = getattr(self, 'item_loader', None)
 		item_loaders = self.item_loaders.copy()
@@ -338,6 +349,8 @@ def yielder(self):
 
 		# Check for sudo requirements and prepare the password if needed
 		sudo_password = self._prompt_sudo(self.cmd)
+		if sudo_password and sudo_password == -1:
+			return
 
 		# Prepare cmds
 		command = self.cmd if self.shell else shlex.split(self.cmd)
@@ -409,22 +422,16 @@ def yielder(self):
 				line = self.run_hooks('on_line', line)
 
 				# Run item_loader to try parsing as dict
-				items = None
+				item_count = 0
 				if self.output_json:
-					items = self.run_item_loaders(line)
+					for item in self.run_item_loaders(line):
+						yield item
+						item_count += 1
 
-				# Yield line if no items parsed
-				if not items:
+				# Yield line if no items were yielded
+				if item_count == 0:
 					yield line
 
-				# Turn results into list if not already a list
-				elif not isinstance(items, list):
-					items = [items]
-
-				# Yield items
-				if items:
-					yield from items
-
 		except KeyboardInterrupt:
 			self.process.kill()
 			self.killed = True
@@ -433,19 +440,16 @@ def yielder(self):
 		self._wait_for_end()
 
 	def run_item_loaders(self, line):
-		"""Run item loaders on a string."""
-		items = []
+		"""Run item loaders against an output line."""
 		for item_loader in self.item_loaders:
-			result = None
 			if (callable(item_loader)):
-				result = item_loader(self, line)
+				yield from item_loader(self, line)
 			elif item_loader:
-				result = item_loader.run(line)
-			if isinstance(result, dict):
-				result = [result]
-			if result:
-				items.extend(result)
-		return items
+				name = item_loader.__class__.__name__.replace('Serializer', '').lower()
+				default_callback = lambda self, x: [(yield x)]  # noqa: E731
+				callback = getattr(self, f'on_{name}_loaded', None) or default_callback
+				for item in item_loader.run(line):
+					yield from callback(self, item)
 
 	def _prompt_sudo(self, command):
 		"""
@@ -464,13 +468,18 @@ def _prompt_sudo(self, command):
 			return None
 
 		# Check if sudo can be executed without a password
-		if subprocess.run(['sudo', '-n', 'true'], capture_output=True).returncode == 0:
-			return None
+		try:
+			if subprocess.run(['sudo', '-n', 'true'], capture_output=False).returncode == 0:
+				return None
+		except ValueError:
+			error = "Could not run sudo check test"
+			self.errors.append(error)
 
 		# Check if we have a tty
 		if not os.isatty(sys.stdin.fileno()):
-			self._print("No TTY detected. Sudo password prompt requires a TTY to proceed.", color='bold red')
-			sys.exit(1)
+			error = "No TTY detected. Sudo password prompt requires a TTY to proceed."
+			self.errors.append(error)
+			return -1
 
 		# If not, prompt the user for a password
 		self._print('[bold red]Please enter sudo password to continue.[/]')
@@ -486,8 +495,9 @@ def _prompt_sudo(self, command):
 			if result.returncode == 0:
 				return sudo_password  # Password is correct
 			self._print("Sorry, try again.")
-		self._print("Sudo password verification failed after 3 attempts.")
-		return None
+		error = "Sudo password verification failed after 3 attempts."
+		self.errors.append(error)
+		return -1
 
 	def _wait_for_end(self):
 		"""Wait for process to finish and process output and return code."""
@@ -576,8 +586,16 @@ def _process_opts(
 
 		return opts_str.strip()
 
+	@staticmethod
+	def _get_opt_default(opt_name, opts_conf):
+		for k, v in opts_conf.items():
+			if k == opt_name:
+				return v.get('default', None)
+		return None
+
 	@staticmethod
 	def _get_opt_value(opts, opt_name, opts_conf={}, opt_prefix='', default=None):
+		default = default or Command._get_opt_default(opt_name, opts_conf)
 		aliases = [
 			opts.get(f'{opt_prefix}_{opt_name}'),
 			opts.get(f'{opt_prefix}.{opt_name}'),

secator/serializers/json.py

@@ -7,9 +7,9 @@ def run(self, line):
 		start_index = line.find('{')
 		end_index = line.rfind('}')
 		if start_index == -1 or end_index == -1:
-			return None
+			return
 		try:
 			json_obj = line[start_index:end_index+1]
-			return yaml.safe_load(json_obj)
+			yield yaml.safe_load(json_obj)
 		except yaml.YAMLError:
-			return None
+			return

secator/serializers/regex.py

@@ -11,7 +11,7 @@ def run(self, line):
 		match = self.regex.match(line)
 		output = {}
 		if not match:
-			return None
+			return
 		for field in self.fields:
 			output[field] = match.group(field)
-		return output
+		yield output