diff --git a/CHANGELOG.md b/CHANGELOG.md index deeadda5..3b689e16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,49 @@ # Changelog +## [0.6.0](https://github.com/freelabz/secator/compare/v0.5.2...v0.6.0) (2024-07-25) + + +### Features + +* add duplicate finder to mongodb hooks ([#409](https://github.com/freelabz/secator/issues/409)) ([fb0e11c](https://github.com/freelabz/secator/commit/fb0e11cd2b64bf51bc862f47243c8c0602d3d5e9)) +* basic helm chart ([#408](https://github.com/freelabz/secator/issues/408)) ([6b2f84f](https://github.com/freelabz/secator/commit/6b2f84f61bd8eccf2cdd61b6ffdc2eb4489240bc)) + + +### Bug Fixes + +* Dockerfile broken apt install ([#407](https://github.com/freelabz/secator/issues/407)) ([c023279](https://github.com/freelabz/secator/commit/c02327968ecea816004636801684b336735df439)) +* **tasks:** duplicate meta opt entry ([#401](https://github.com/freelabz/secator/issues/401)) ([ae56aa6](https://github.com/freelabz/secator/commit/ae56aa62f5a18936a1787547e37bbe636e6e43c3)) + +## [0.5.2](https://github.com/freelabz/secator/compare/v0.5.1...v0.5.2) (2024-05-07) + + +### Bug Fixes + +* **nuclei,katana:** add -sr flag and write http responses and screenshot to correct folder ([#395](https://github.com/freelabz/secator/issues/395)) ([1a51790](https://github.com/freelabz/secator/commit/1a51790c9231f593631c2780b6d5e0fa89f1aa55)) + +## [0.5.1](https://github.com/freelabz/secator/compare/v0.5.0...v0.5.1) (2024-05-06) + + +### Bug Fixes + +* **output:** add headers to Url and print HTTP method when not GET ([#390](https://github.com/freelabz/secator/issues/390)) ([5a87d7b](https://github.com/freelabz/secator/commit/5a87d7b8bc1dd098999f3864952e98068fd32efc)) +* **report:** do not remove duplicate in reports by default ([#392](https://github.com/freelabz/secator/issues/392)) ([7d74ae8](https://github.com/freelabz/secator/commit/7d74ae80bfd99c31714a5e7e25f2bd1caa642eb4)) + +## [0.5.0](https://github.com/freelabz/secator/compare/v0.4.1...v0.5.0) (2024-05-03) + + +### Features + +* add searchsploit output fields ([#278](https://github.com/freelabz/secator/issues/278)) ([00872c4](https://github.com/freelabz/secator/commit/00872c4a7f9b1ec76ee1bfd7a00919d53cbdb30a)) +* **cli:** add report list / export commands ([#367](https://github.com/freelabz/secator/issues/367)) ([ab396a3](https://github.com/freelabz/secator/commit/ab396a3098c6d4c46cf9c9b29bd5c54579421646)) +* **config:** load external tasks from template dir ([#373](https://github.com/freelabz/secator/issues/373)) ([0c63c02](https://github.com/freelabz/secator/commit/0c63c02c8eca477a6752f4af466c4303801019de)) + + +### Bug Fixes + +* **cli:** catch JSON parse errors ([#378](https://github.com/freelabz/secator/issues/378)) ([5e3d7f2](https://github.com/freelabz/secator/commit/5e3d7f2d2938a857e7599a429a6cfabf3b12347b)) +* **nmap:** resolve -sS tcp syn stealth issue ([#376](https://github.com/freelabz/secator/issues/376)) ([a3efc65](https://github.com/freelabz/secator/commit/a3efc651dfa4d8fa34d611b9aea2e156352fdc45)) + ## [0.4.1](https://github.com/freelabz/secator/compare/v0.4.0...v0.4.1) (2024-04-30) diff --git a/Dockerfile b/Dockerfile index 3bca8404..54704348 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,7 @@ RUN apt update -y && \ jq \ openssl \ proxychains \ - proxychains-ng \ - && rm -rf /var/lib/apt/lists/* + proxychains-ng # Install Metasploit framework RUN curl https://raw.githubusercontent.com/rapid7/metasploit-omnibus/master/config/templates/metasploit-framework-wrappers/msfupdate.erb > msfinstall @@ -43,5 +42,8 @@ RUN secator install addons mongodb RUN secator install addons redis RUN secator install addons dev +# Cleanup +RUN rm -rf /var/lib/apt/lists/* + # Set entrypoint ENTRYPOINT ["secator"] \ No newline at end of file diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 00000000..8f367896 --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: Secator +description: A Secator Helm chart for Kubernetes + +type: application +version: 0.1.0 +appVersion: "0.5.2" diff --git a/helm/templates/redis-service.yaml b/helm/templates/redis-service.yaml new file mode 100644 index 00000000..f396ffac --- /dev/null +++ b/helm/templates/redis-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.redis.name }} + namespace: {{ .Values.namespace }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.redis.port }} + name: client + selector: + app: redis \ No newline at end of file diff --git a/helm/templates/redis.yaml b/helm/templates/redis.yaml new file mode 100644 index 00000000..a8ef0c98 --- /dev/null +++ b/helm/templates/redis.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Values.redis.name }} + namespace: {{ .Values.namespace }} +spec: + selector: + matchLabels: + app: redis + serviceName: {{ .Values.redis.name }} + replicas: {{ .Values.redis.replicas }} + template: + metadata: + labels: + app: redis + spec: + containers: + - name: {{ .Values.redis.name }} + image: {{ .Values.redis.image }} + ports: + - containerPort: {{ .Values.redis.port }} + name: client \ No newline at end of file diff --git a/helm/templates/secator-manager.yaml b/helm/templates/secator-manager.yaml new file mode 100644 index 00000000..b9e3acaa --- /dev/null +++ b/helm/templates/secator-manager.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.secatorManager.name }} + namespace: {{ .Values.namespace }} + labels: + name: secator-manager +spec: + containers: + - name: secator-manager + image: {{ .Values.secatorManager.image }} + command: ["tail"] + args: ["-F", "anything"] + env: + - name: SECATOR_CELERY_BROKER_URL + value: "redis://{{ .Values.redis.name }}:6379/0" + - name: SECATOR_CELERY_RESULT_BACKEND + value: "redis://{{ .Values.redis.name }}:6379/0" diff --git a/helm/templates/secator-worker.yaml b/helm/templates/secator-worker.yaml new file mode 100644 index 00000000..b540ffe2 --- /dev/null +++ b/helm/templates/secator-worker.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.secatorWorker.name }} + namespace: {{ .Values.namespace }} +spec: + selector: + matchLabels: + app: secator-worker + template: + metadata: + labels: + app: secator-worker + spec: + containers: + - name: {{ .Values.secatorWorker.name }} + image: {{ .Values.secatorWorker.image }} + command: ["secator"] + args: ["worker"] + env: + - name: SECATOR_CELERY_BROKER_URL + value: "redis://{{ .Values.redis.name }}:6379/0" + - name: SECATOR_CELERY_RESULT_BACKEND + value: "redis://{{ .Values.redis.name }}:6379/0" \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 00000000..72d731c8 --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,34 @@ +# Default values for Secator +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +namespace: secator + +secatorManager: + name: secator-manager + image: "freelabz/secator" + + # Empty if using default repository + repository: + # Empty if using tag "latest" + tag: + +secatorWorker: + name: secator-worker + image: "freelabz/secator" + + # Empty if using default repository + repository: + # Empty if using tag "latest" + tag: + +redis: + name: redis + image: "redis" + + # Empty if using default repository + repository: + # Empty if using tag "latest" + tag: + replicas: 1 + port: 6379 diff --git a/package.json b/package.json new file mode 100644 index 00000000..6ff934af --- /dev/null +++ b/package.json @@ -0,0 +1,29 @@ +{ + "name": "secator-worker", + "version": "0.0.1", + "description": "Secator worker", + "main": "", + "scripts": { + "solo": "npm run venv && venv/bin/secator worker -r", + "dev": "npm run venv && SECATOR_CELERY_BROKER_URL=redis://localhost:6379 SECATOR_CELERY_RESULT_BACKEND=redis://localhost:6379 venv/bin/secator worker -r", + "venv": "pip install virtualenv --break-system-packages && virtualenv venv && chmod +x venv/bin/activate && . venv/bin/activate && venv/bin/pip install -e .[dev,worker,redis,mongodb,trace]", + "generate": "rm -r venv && npm run venv && venv/bin/pip install fastapi uvicorn && venv/bin/pip freeze > requirements.txt", + "docker:build": "docker build -t secator .", + "docker:push": "gcloud builds submit .", + "docker:logs": "docker logs -f secator", + "docker:start": "docker run --name=secator-worker --network=host -e SECATOR_CELERY_BROKER_URL=redis://localhost:6379 -e SECATOR_CELERY_RESULT_BACKEND=redis://localhost:6379 -d secator worker", + "docker:stop": "docker stop secator-worker; docker rm secator-worker", + "docker:start-redis": "docker run --name redis -p 6379:6379 -d redis", + "docker:stop-redis": "docker stop redis; docker rm redis", + "docker:start-mongodb": "docker run --name mongodb -p 27017:27017 -d mongo:latest", + "docker:stop-mongodb": "docker stop mongodb; docker rm mongodb", + "docker:start-deps": "npm run docker:start-mongodb; npm run docker:start-redis", + "docker:stop-deps": "npm run docker:stop-mongodb; npm run docker:stop-redis", + "docker:start-all": "npm run docker:start-deps; npm run docker:start", + "docker:stop-all": "npm run docker:stop-deps; npm run docker:stop" + }, + "keywords": [], + "author": "ocervello@freelabz.com", + "license": "", + "dependencies": {} +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 914281b9..f0dfa6ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "secator" -version = "0.4.1" +version = "0.6.0" authors = [{ name = "FreeLabz", email = "sales@freelabz.com" }] readme = "README.md" description = "The pentester's swiss knife." @@ -30,7 +30,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "bs4 < 1", + "beautifulsoup4 <= 5", 'celery < 6', "cpe < 2", "dotmap < 2", @@ -46,6 +46,7 @@ dependencies = [ "requests < 3", "rich < 14", "rich-click < 1.7", + "tldextract < 6", "typing_extensions < 5", "validators < 1", "xmltodict < 1" diff --git a/secator/celery.py b/secator/celery.py index 6f3f0e95..bc06938c 100644 --- a/secator/celery.py +++ b/secator/celery.py @@ -1,11 +1,10 @@ import gc import logging import traceback -from time import sleep from celery import Celery, chain, chord, signals from celery.app import trace -from celery.result import AsyncResult, allow_join_result +from celery.result import allow_join_result # from pyinstrument import Profiler # TODO: make pyinstrument optional from rich.logging import RichHandler @@ -146,8 +145,7 @@ def break_task(task_cls, task_opts, targets, results=[], chunk_size=1): @app.task(bind=True) def run_task(self, args=[], kwargs={}): - if CONFIG.debug.level > 1: - logger.info(f'Received task with args {args} and kwargs {kwargs}') + debug(f'Received task with args {args} and kwargs {kwargs}', sub="celery", level=2) if 'context' not in kwargs: kwargs['context'] = {} kwargs['context']['celery_id'] = self.request.id @@ -157,8 +155,7 @@ def run_task(self, args=[], kwargs={}): @app.task(bind=True) def run_workflow(self, args=[], kwargs={}): - if CONFIG.debug.level > 1: - logger.info(f'Received workflow with args {args} and kwargs {kwargs}') + debug(f'Received workflow with args {args} and kwargs {kwargs}', sub="celery", level=2) if 'context' not in kwargs: kwargs['context'] = {} kwargs['context']['celery_id'] = self.request.id @@ -168,8 +165,7 @@ def run_workflow(self, args=[], kwargs={}): @app.task(bind=True) def run_scan(self, args=[], kwargs={}): - if CONFIG.debug.level > 1: - logger.info(f'Received scan with args {args} and kwargs {kwargs}') + debug(f'Received scan with args {args} and kwargs {kwargs}', sub="celery", level=2) if 'context' not in kwargs: kwargs['context'] = {} kwargs['context']['celery_id'] = self.request.id @@ -354,56 +350,9 @@ def forward_results(results): results = deduplicate(results, attr='_uuid') return results - -#---------------------# -# Celery result utils # -#---------------------# - - -def poll_task(result, seen=[]): - """Poll Celery result tree recursively to get results live. - - TODO: function is incomplete, as it does not parse all results. - - Args: - result (Union[AsyncResult, GroupResult]): Celery result object. - seen (list): List of seen results (do not yield again). - - Yields: - dict: Result. - """ - if result is None: - return - - if result.children: - for child in result.children: - yield from poll_task(child, seen=seen) - else: - res = AsyncResult(result.id) - if not res.info: - sleep(0.1) - yield from poll_task(result, seen=seen) - - # Task done running - if isinstance(res.info, list): - for item in res.info: - if item._uuid not in seen: - yield res.id, None, item - seen.append(item._uuid) - return - - # Get task partial results, remove duplicates - results = res.info['results'] - name = res.info['name'] - for item in results: - if item._uuid not in seen: - yield res.id, name, item - seen.append(item._uuid) - - # Task still running, keep polling - if not res.ready(): - sleep(0.1) - yield from poll_task(result, seen=seen) +#--------------# +# Celery utils # +#--------------# def is_celery_worker_alive(): diff --git a/secator/cli.py b/secator/cli.py index 04f08494..3ea78115 100644 --- a/secator/cli.py +++ b/secator/cli.py @@ -561,14 +561,17 @@ def report_list(workspace): reports_dir = CONFIG.dirs.reports json_reports = reports_dir.glob("**/**/report.json") ws_reports = {} - for report in json_reports: - ws, runner, number = str(report).split('/')[-4:-1] + for path in json_reports: + ws, runner, number = str(path).split('/')[-4:-1] if ws not in ws_reports: ws_reports[ws] = [] - with open(report, 'r') as f: - content = json.loads(f.read()) - data = {'path': report, 'name': content['info']['name'], 'runner': runner} - ws_reports[ws].append(data) + with open(path, 'r') as f: + try: + content = json.loads(f.read()) + data = {'path': path, 'name': content['info']['name'], 'runner': runner} + ws_reports[ws].append(data) + except json.JSONDecodeError as e: + console.print(f'[bold red]Could not load {path}: {str(e)}') for ws in ws_reports: if workspace and not ws == workspace: diff --git a/secator/config.py b/secator/config.py index 0f6c179e..57277a65 100644 --- a/secator/config.py +++ b/secator/config.py @@ -74,6 +74,7 @@ class Runners(StrictModel): progress_update_frequency: int = 60 skip_cve_search: bool = False skip_cve_low_confidence: bool = True + remove_duplicates: bool = False class HTTP(StrictModel): diff --git a/secator/configs/workflows/host_recon.yaml b/secator/configs/workflows/host_recon.yaml index e775492b..7edb8ccb 100644 --- a/secator/configs/workflows/host_recon.yaml +++ b/secator/configs/workflows/host_recon.yaml @@ -9,8 +9,11 @@ input_types: tasks: naabu: description: Find open ports + ports: "-" # scan all ports nmap: description: Search for vulnerabilities on open ports + skip_host_discovery: True + version_detection: True targets_: port.host ports_: port.port httpx: @@ -38,4 +41,4 @@ results: # condition: item.confidence == 'high' - type: url - condition: item.status_code != 0 \ No newline at end of file + condition: item.status_code != 0 diff --git a/secator/configs/workflows/port_scan.yaml b/secator/configs/workflows/port_scan.yaml index 2c7762b2..54040129 100644 --- a/secator/configs/workflows/port_scan.yaml +++ b/secator/configs/workflows/port_scan.yaml @@ -5,11 +5,15 @@ description: Port scan tags: [recon, network, http, vuln] input_types: - host + - cidr_range tasks: naabu: description: Find open ports + ports: "-" # scan all ports nmap: description: Search for vulnerabilities on open ports + skip_host_discovery: True + version_detection: True targets_: port.host ports_: port.port _group: @@ -31,4 +35,4 @@ results: - type: url condition: item.status_code != 0 - - type: vulnerability \ No newline at end of file + - type: vulnerability diff --git a/secator/decorators.py b/secator/decorators.py index 114dad0f..42ebd2c2 100644 --- a/secator/decorators.py +++ b/secator/decorators.py @@ -269,9 +269,17 @@ def func(ctx, **opts): driver = opts.pop('driver', '') show = opts['show'] context = {'workspace_name': ws} + + # Remove options whose values are default values + for k, v in options.items(): + opt_name = k.replace('-', '_') + if opt_name in opts and opts[opt_name] == v.get('default', None): + del opts[opt_name] + # TODO: maybe allow this in the future # unknown_opts = get_unknown_opts(ctx) # opts.update(unknown_opts) + targets = opts.pop(input_type) targets = expand_input(targets) if sync or show: diff --git a/secator/definitions.py b/secator/definitions.py index ed00dd11..88532c20 100644 --- a/secator/definitions.py +++ b/secator/definitions.py @@ -54,6 +54,7 @@ HEADER = 'header' HOST = 'host' IP = 'ip' +PROTOCOL = 'protocol' LINES = 'lines' METHOD = 'method' MATCH_CODES = 'match_codes' diff --git a/secator/hooks/mongodb.py b/secator/hooks/mongodb.py index df557252..a1443456 100644 --- a/secator/hooks/mongodb.py +++ b/secator/hooks/mongodb.py @@ -206,6 +206,6 @@ def tag_duplicates(ws_id: str = None): 'on_item': [update_finding], 'on_duplicate': [update_finding], 'on_iter': [update_runner], - 'on_end': [update_runner] + 'on_end': [update_runner, find_duplicates] } } diff --git a/secator/output_types/exploit.py b/secator/output_types/exploit.py index 313380a2..69c7ebb3 100644 --- a/secator/output_types/exploit.py +++ b/secator/output_types/exploit.py @@ -34,6 +34,9 @@ class Exploit(OutputType): ] _sort_by = ('matched_at', 'name') + def __str__(self): + return self.name + def __repr__(self): s = f'[bold red]⍼[/] \[[bold red]{self.name}' if self.reference: diff --git a/secator/output_types/ip.py b/secator/output_types/ip.py index 6b39e835..dc1f85a0 100644 --- a/secator/output_types/ip.py +++ b/secator/output_types/ip.py @@ -1,16 +1,23 @@ import time from dataclasses import dataclass, field +from enum import Enum from secator.definitions import ALIVE, IP from secator.output_types import OutputType from secator.utils import rich_to_ansi +class IpProtocol(str, Enum): + IPv6 = 'IPv6' + IPv4 = 'IPv4' + + @dataclass class Ip(OutputType): ip: str host: str = '' alive: bool = False + protocol: str = field(default=IpProtocol.IPv4) _source: str = field(default='', repr=True) _type: str = field(default='ip', repr=True) _timestamp: int = field(default_factory=lambda: time.time(), compare=False) diff --git a/secator/output_types/port.py b/secator/output_types/port.py index 7b108759..aba0f2f7 100644 --- a/secator/output_types/port.py +++ b/secator/output_types/port.py @@ -14,6 +14,7 @@ class Port(OutputType): service_name: str = field(default='', compare=False) cpes: list = field(default_factory=list, compare=False) host: str = field(default='', repr=True, compare=False) + protocol: str = field(default='TCP', repr=True, compare=False) extra_data: dict = field(default_factory=dict, compare=False) _timestamp: int = field(default_factory=lambda: time.time(), compare=False) _source: str = field(default='', repr=True, compare=False) @@ -38,6 +39,8 @@ def __str__(self) -> str: def __repr__(self) -> str: s = f'🔓 {self.ip}:[bold red]{self.port:<4}[/] [bold yellow]{self.state.upper()}[/]' + if self.protocol != 'TCP': + s += f' \[[yellow3]{self.protocol}[/]]' if self.service_name: s += f' \[[bold purple]{self.service_name}[/]]' if self.host: diff --git a/secator/output_types/progress.py b/secator/output_types/progress.py index 1be7a954..677d26e4 100644 --- a/secator/output_types/progress.py +++ b/secator/output_types/progress.py @@ -8,7 +8,7 @@ @dataclass class Progress(OutputType): duration: str - percent: int + percent: int = 0 errors: list = field(default_factory=list) extra_data: dict = field(default_factory=dict) _source: str = field(default='', repr=True) diff --git a/secator/output_types/url.py b/secator/output_types/url.py index c6bd32d7..8ed79787 100644 --- a/secator/output_types/url.py +++ b/secator/output_types/url.py @@ -23,6 +23,7 @@ class Url(OutputType): lines: int = field(default=0, compare=False) screenshot_path: str = field(default='', compare=False) stored_response_path: str = field(default='', compare=False) + headers: dict = field(default_factory=dict, repr=True, compare=False) _source: str = field(default='', repr=True, compare=False) _type: str = field(default='url', repr=True) _timestamp: int = field(default_factory=lambda: time.time(), compare=False) @@ -55,6 +56,8 @@ def __str__(self): def __repr__(self): s = f'🔗 [white]{self.url}' + if self.method and self.method != 'GET': + s += f' \[[turquoise4]{self.method}[/]]' if self.status_code and self.status_code != 0: if self.status_code < 400: s += f' \[[green]{self.status_code}[/]]' diff --git a/secator/report.py b/secator/report.py index 69b3b2da..5e4fba39 100644 --- a/secator/report.py +++ b/secator/report.py @@ -1,5 +1,6 @@ import operator +from secator.config import CONFIG from secator.output_types import OUTPUT_TYPES, OutputType from secator.utils import merge_opts, get_file_timestamp from secator.rich import console @@ -48,6 +49,7 @@ def build(self): 'title': self.title, 'runner': self.runner.__class__.__name__, 'name': self.runner.config.name, + 'status': self.runner.status, 'targets': self.runner.targets, 'total_time': str(self.runner.elapsed), 'total_human': self.runner.elapsed_human, @@ -64,8 +66,10 @@ def build(self): sort_by, _ = get_table_fields(output_type) items = [ item for item in self.runner.results - if isinstance(item, OutputType) and item._type == output_name and not item._duplicate + if isinstance(item, OutputType) and item._type == output_name ] + if CONFIG.runners.remove_duplicates: + items = [item for item in items if not item._duplicate] if items: if sort_by and all(sort_by): items = sorted(items, key=operator.attrgetter(*sort_by)) diff --git a/secator/runners/_base.py b/secator/runners/_base.py index 573cbbed..9412fd37 100644 --- a/secator/runners/_base.py +++ b/secator/runners/_base.py @@ -29,14 +29,11 @@ HOOKS = [ 'before_init', 'on_init', - 'on_start', 'on_end', 'on_item_pre_convert', 'on_item', 'on_duplicate', - 'on_line', 'on_iter', - 'on_error', ] VALIDATORS = [ @@ -158,10 +155,10 @@ def __init__(self, config, targets, results=[], run_opts={}, hooks={}, context={ # Hooks self.raise_on_error = self.run_opts.get('raise_on_error', False) - self.hooks = {name: [] for name in HOOKS} + self.hooks = {name: [] for name in HOOKS + getattr(self, 'hooks', [])} for key in self.hooks: - # Register class specific hooks + # Register class + derived class hooks class_hook = getattr(self, key, None) if class_hook: name = f'{self.__class__.__name__}.{key}' @@ -297,7 +294,7 @@ def mark_duplicates(self): debug('running duplicate check', id=self.config.name, sub='runner.mark_duplicates') dupe_count = 0 for item in self.results: - debug('running duplicate check', obj=item.toDict(), obj_breaklines=True, sub='runner.mark_duplicates', level=5) + # debug('running duplicate check', obj=item.toDict(), obj_breaklines=True, sub='runner.mark_duplicates', level=5) others = [f for f in self.results if f == item and f._uuid != item._uuid] if others: main = max(item, *others) @@ -306,6 +303,7 @@ def mark_duplicates(self): main._related.extend([dupe._uuid for dupe in dupes]) main._related = list(dict.fromkeys(main._related)) if main._uuid != item._uuid: + debug(f'found {len(others)} duplicates for', obj=item.toDict(), obj_breaklines=True, sub='runner.mark_duplicates', level=5) # noqa: E501 item._duplicate = True item = self.run_hooks('on_item', item) if item._uuid not in main._related: @@ -839,12 +837,16 @@ def _process_item(self, item: dict): if not self.run_validators('item', item): return None - # Run item hooks - item = self.run_hooks('on_item_pre_convert', item) - if not item: - return None - # Convert output dict to another schema + if isinstance(item, dict): + item = self.run_hooks('on_item_pre_convert', item) + if not item: + return None + if not self.orig: + item = self._convert_item_schema(item) + else: + item = DotMap(item) + if isinstance(item, dict) and not self.orig: item = self._convert_item_schema(item) elif isinstance(item, OutputType): diff --git a/secator/runners/command.py b/secator/runners/command.py index 6104769c..fecd5bbc 100644 --- a/secator/runners/command.py +++ b/secator/runners/command.py @@ -82,6 +82,14 @@ class Command(Runner): item_loader = None item_loaders = [JSONSerializer(),] + # Hooks + hooks = [ + 'on_start', + 'on_cmd', + 'on_line', + 'on_error', + ] + # Ignore return code ignore_return_code = False @@ -141,6 +149,9 @@ def __init__(self, input=None, **run_opts): # Build command self._build_cmd() + # Run on_cmd hook + self.run_hooks('on_cmd') + # Build item loaders instance_func = getattr(self, 'item_loader', None) item_loaders = self.item_loaders.copy() @@ -338,6 +349,8 @@ def yielder(self): # Check for sudo requirements and prepare the password if needed sudo_password = self._prompt_sudo(self.cmd) + if sudo_password and sudo_password == -1: + return # Prepare cmds command = self.cmd if self.shell else shlex.split(self.cmd) @@ -409,22 +422,16 @@ def yielder(self): line = self.run_hooks('on_line', line) # Run item_loader to try parsing as dict - items = None + item_count = 0 if self.output_json: - items = self.run_item_loaders(line) + for item in self.run_item_loaders(line): + yield item + item_count += 1 - # Yield line if no items parsed - if not items: + # Yield line if no items were yielded + if item_count == 0: yield line - # Turn results into list if not already a list - elif not isinstance(items, list): - items = [items] - - # Yield items - if items: - yield from items - except KeyboardInterrupt: self.process.kill() self.killed = True @@ -433,19 +440,16 @@ def yielder(self): self._wait_for_end() def run_item_loaders(self, line): - """Run item loaders on a string.""" - items = [] + """Run item loaders against an output line.""" for item_loader in self.item_loaders: - result = None if (callable(item_loader)): - result = item_loader(self, line) + yield from item_loader(self, line) elif item_loader: - result = item_loader.run(line) - if isinstance(result, dict): - result = [result] - if result: - items.extend(result) - return items + name = item_loader.__class__.__name__.replace('Serializer', '').lower() + default_callback = lambda self, x: [(yield x)] # noqa: E731 + callback = getattr(self, f'on_{name}_loaded', None) or default_callback + for item in item_loader.run(line): + yield from callback(self, item) def _prompt_sudo(self, command): """ @@ -464,13 +468,18 @@ def _prompt_sudo(self, command): return None # Check if sudo can be executed without a password - if subprocess.run(['sudo', '-n', 'true'], capture_output=True).returncode == 0: - return None + try: + if subprocess.run(['sudo', '-n', 'true'], capture_output=False).returncode == 0: + return None + except ValueError: + error = "Could not run sudo check test" + self.errors.append(error) # Check if we have a tty if not os.isatty(sys.stdin.fileno()): - self._print("No TTY detected. Sudo password prompt requires a TTY to proceed.", color='bold red') - sys.exit(1) + error = "No TTY detected. Sudo password prompt requires a TTY to proceed." + self.errors.append(error) + return -1 # If not, prompt the user for a password self._print('[bold red]Please enter sudo password to continue.[/]') @@ -486,8 +495,9 @@ def _prompt_sudo(self, command): if result.returncode == 0: return sudo_password # Password is correct self._print("Sorry, try again.") - self._print("Sudo password verification failed after 3 attempts.") - return None + error = "Sudo password verification failed after 3 attempts." + self.errors.append(error) + return -1 def _wait_for_end(self): """Wait for process to finish and process output and return code.""" @@ -576,8 +586,16 @@ def _process_opts( return opts_str.strip() + @staticmethod + def _get_opt_default(opt_name, opts_conf): + for k, v in opts_conf.items(): + if k == opt_name: + return v.get('default', None) + return None + @staticmethod def _get_opt_value(opts, opt_name, opts_conf={}, opt_prefix='', default=None): + default = default or Command._get_opt_default(opt_name, opts_conf) aliases = [ opts.get(f'{opt_prefix}_{opt_name}'), opts.get(f'{opt_prefix}.{opt_name}'), diff --git a/secator/serializers/json.py b/secator/serializers/json.py index efca544e..7a624d92 100644 --- a/secator/serializers/json.py +++ b/secator/serializers/json.py @@ -7,9 +7,9 @@ def run(self, line): start_index = line.find('{') end_index = line.rfind('}') if start_index == -1 or end_index == -1: - return None + return try: json_obj = line[start_index:end_index+1] - return yaml.safe_load(json_obj) + yield yaml.safe_load(json_obj) except yaml.YAMLError: - return None + return diff --git a/secator/serializers/regex.py b/secator/serializers/regex.py index ccb1a153..f875d9e0 100644 --- a/secator/serializers/regex.py +++ b/secator/serializers/regex.py @@ -11,7 +11,7 @@ def run(self, line): match = self.regex.match(line) output = {} if not match: - return None + return for field in self.fields: output[field] = match.group(field) - return output + yield output diff --git a/secator/tasks/_categories.py b/secator/tasks/_categories.py index eb34fa44..b3f0d9e6 100644 --- a/secator/tasks/_categories.py +++ b/secator/tasks/_categories.py @@ -44,7 +44,7 @@ ] OPTS_HTTP_CRAWLERS = OPTS_HTTP + [ - DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT, + DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS, MATCH_CODES ] diff --git a/secator/tasks/bup.py b/secator/tasks/bup.py new file mode 100644 index 00000000..427b5e09 --- /dev/null +++ b/secator/tasks/bup.py @@ -0,0 +1,86 @@ +import re + +from secator.decorators import task +from secator.output_types import Url +from secator.tasks._categories import Http +from secator.definitions import ( + HEADER, DELAY, FOLLOW_REDIRECT, METHOD, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, USER_AGENT, + DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS, + MATCH_CODES, OPT_NOT_SUPPORTED, URL +) + + +@task() +class bup(Http): + cmd = 'bup' + input_flag = '-u' + input_type = URL + json_flag = '--jsonl' + opt_prefix = '--' + opts = { + 'spoofport': {'type': int, 'short': 'sp', 'help': 'Port(s) to inject in port-specific headers'}, + 'spoofip': {'type': str, 'short': 'si', 'help': 'IP(s) to inject in ip-specific headers'}, + 'mode': {'type': str, 'help': 'Bypass modes.'}, + } + opt_key_map = { + HEADER: 'header', + DELAY: OPT_NOT_SUPPORTED, + FOLLOW_REDIRECT: OPT_NOT_SUPPORTED, + METHOD: OPT_NOT_SUPPORTED, + RATE_LIMIT: OPT_NOT_SUPPORTED, + RETRIES: 'retry', + THREADS: 'threads', + TIMEOUT: 'timeout', + USER_AGENT: OPT_NOT_SUPPORTED, + DEPTH: OPT_NOT_SUPPORTED, + MATCH_REGEX: OPT_NOT_SUPPORTED, + MATCH_SIZE: OPT_NOT_SUPPORTED, + MATCH_WORDS: OPT_NOT_SUPPORTED, + FILTER_REGEX: OPT_NOT_SUPPORTED, + FILTER_CODES: OPT_NOT_SUPPORTED, + FILTER_SIZE: OPT_NOT_SUPPORTED, + FILTER_WORDS: OPT_NOT_SUPPORTED, + FOLLOW_REDIRECT: OPT_NOT_SUPPORTED, + MATCH_CODES: OPT_NOT_SUPPORTED, + PROXY: OPT_NOT_SUPPORTED, + } + output_types = [Url] + output_map = { + Url: { + 'url': 'request_url', + 'method': lambda x: bup.method_extractor(x), + 'headers': lambda x: bup.headers_extractor(x), + 'status_code': 'response_status_code', + 'content_type': 'response_content_type', + 'content_length': 'response_content_length', + 'title': 'response_title', + 'server': 'response_server_type', + 'lines': 'response_lines_count', + 'words': 'response_words_count', + 'stored_response_path': 'response_html_filename', + } + } + install_cmd = 'pipx install bypass-url-parser && pipx upgrade bypass-url-parser' + + @staticmethod + def on_init(self): + self.cmd += f' -o {self.reports_folder}/.outputs/response' + + @staticmethod + def method_extractor(item): + payload = item['request_curl_payload'] + match = re.match(r'-X\s+(\w+)', payload) + if match: + return match.group(1) + return 'GET' + + @staticmethod + def headers_extractor(item): + headers_list = item['response_headers'].split('\n')[1:] + headers = {} + for header in headers_list: + split_headers = header.split(':') + key = split_headers[0] + value = ':'.join(split_headers[1:]) + headers[key] = value + return headers diff --git a/secator/tasks/cariddi.py b/secator/tasks/cariddi.py index 5926a39b..457f9038 100644 --- a/secator/tasks/cariddi.py +++ b/secator/tasks/cariddi.py @@ -1,5 +1,3 @@ -import json - from secator.decorators import task from secator.definitions import (DELAY, DEPTH, FILTER_CODES, FILTER_REGEX, FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT, @@ -41,7 +39,6 @@ class cariddi(HttpCrawler): TIMEOUT: 't', USER_AGENT: 'ua' } - item_loaders = [] install_cmd = 'go install -v github.com/edoardottt/cariddi/cmd/cariddi@latest' install_github_handle = 'edoardottt/cariddi' encoding = 'ansi' @@ -51,53 +48,44 @@ class cariddi(HttpCrawler): profile = 'cpu' @staticmethod - def item_loader(self, line): - items = [] - try: - item = json.loads(line) - url_item = {k: v for k, v in item.items() if k != 'matches'} - url = url_item[URL] - items.append(url_item) - matches = item.get('matches', {}) - params = matches.get('parameters', []) - errors = matches.get('errors', []) - secrets = matches.get('secrets', []) - infos = matches.get('infos', []) - - for param in params: - param_name = param['name'] - for attack in param['attacks']: - extra_data = {'param': param_name, 'source': 'url'} - item = { - 'name': attack + ' param', - 'match': url, - 'extra_data': extra_data - } - items.append(item) - - for error in errors: - match = error['match'] - match = (match[:1000] + '...TRUNCATED') if len(match) > 1000 else match # truncate as this can be a very long match - error['extra_data'] = {'error': match, 'source': 'body'} - error['match'] = url - items.append(error) + def on_json_loaded(self, item): + url_item = {k: v for k, v in item.items() if k != 'matches'} + yield Url(**url_item) + url = url_item[URL] + matches = item.get('matches', {}) + params = matches.get('parameters', []) + errors = matches.get('errors', []) + secrets = matches.get('secrets', []) + infos = matches.get('infos', []) - for secret in secrets: - match = secret['match'] - secret['extra_data'] = {'secret': match, 'source': 'body'} - secret['match'] = url - items.append(secret) + for param in params: + param_name = param['name'] + for attack in param['attacks']: + extra_data = {'param': param_name, 'source': 'url'} + yield Tag( + name=f'{attack} param', + match=url, + extra_data=extra_data + ) - for info in infos: - CARIDDI_IGNORE_LIST = ['BTC address'] # TODO: make this a config option - if info['name'] in CARIDDI_IGNORE_LIST: - continue - match = info['match'] - info['extra_data'] = {'info': match, 'source': 'body'} - info['match'] = url - items.append(info) + for error in errors: + match = error['match'] + match = (match[:1000] + '...TRUNCATED') if len(match) > 1000 else match # truncate as this can be a very long match + error['extra_data'] = {'error': match, 'source': 'body'} + error['match'] = url + yield Tag(**error) - except json.decoder.JSONDecodeError: - pass + for secret in secrets: + match = secret['match'] + secret['extra_data'] = {'secret': match, 'source': 'body'} + secret['match'] = url + yield Tag(**secret) - return items + for info in infos: + CARIDDI_IGNORE_LIST = ['BTC address'] # TODO: make this a config option + if info['name'] in CARIDDI_IGNORE_LIST: + continue + match = info['match'] + info['extra_data'] = {'info': match, 'source': 'body'} + info['match'] = url + yield Tag(**info) diff --git a/secator/tasks/dalfox.py b/secator/tasks/dalfox.py index 08eef6a6..1c829477 100644 --- a/secator/tasks/dalfox.py +++ b/secator/tasks/dalfox.py @@ -23,6 +23,7 @@ class dalfox(VulnHttp): input_type = URL input_flag = 'url' file_flag = 'file' + input_chunk_size = 1 json_flag = '--format json' version_flag = 'version' opt_prefix = '--' diff --git a/secator/tasks/dnsx.py b/secator/tasks/dnsx.py index 4b7bd1f7..3d007b0c 100644 --- a/secator/tasks/dnsx.py +++ b/secator/tasks/dnsx.py @@ -1,18 +1,19 @@ from secator.decorators import task from secator.definitions import (OPT_PIPE_INPUT, RATE_LIMIT, RETRIES, THREADS) -from secator.output_types import Record +from secator.output_types import Record, Ip, Subdomain +from secator.output_types.ip import IpProtocol from secator.tasks._categories import ReconDns -import json +from secator.utils import extract_domain_info @task() class dnsx(ReconDns): """dnsx is a fast and multi-purpose DNS toolkit designed for running various retryabledns library.""" - cmd = 'dnsx -resp -a -aaaa -cname -mx -ns -txt -srv -ptr -soa -axfr -caa' + cmd = 'dnsx -resp -recon' json_flag = '-json' input_flag = OPT_PIPE_INPUT file_flag = OPT_PIPE_INPUT - output_types = [Record] + output_types = [Record, Ip, Subdomain] opt_key_map = { RATE_LIMIT: 'rate-limit', RETRIES: 'retry', @@ -23,35 +24,51 @@ class dnsx(ReconDns): 'resolver': {'type': str, 'short': 'r', 'help': 'List of resolvers to use (file or comma separated)'}, 'wildcard_domain': {'type': str, 'short': 'wd', 'help': 'Domain name for wildcard filtering'}, } - install_cmd = 'go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest' install_github_handle = 'projectdiscovery/dnsx' profile = 'io' @staticmethod - def item_loader(self, line): - items = [] - try: - item = json.loads(line) - if self.orig: # original dnsx output - return item - host = item['host'] - record_types = ['a', 'aaaa', 'cname', 'mx', 'ns', 'txt', 'srv', 'ptr', 'soa', 'axfr', 'caa'] - for _type in record_types: - values = item.get(_type, []) - for value in values: - name = value - extra_data = {} - if isinstance(value, dict): - name = value['name'] - extra_data = {k: v for k, v in value.items() if k != 'name'} - items.append({ - 'host': host, - 'name': name, - 'type': _type.upper(), - 'extra_data': extra_data - }) - except json.decoder.JSONDecodeError: - pass + def on_json_loaded(self, item): + # Show full DNS response + quiet = self.get_opt_value('quiet') + if not quiet: + all = item['all'] + for line in all: + yield line + yield '\n' - return items + # Loop through record types and yield records + record_types = ['a', 'aaaa', 'cname', 'mx', 'ns', 'txt', 'srv', 'ptr', 'soa', 'axfr', 'caa'] + host = item['host'] + for _type in record_types: + values = item.get(_type, []) + for value in values: + name = value + extra_data = {} + if isinstance(value, dict): + name = value['name'] + extra_data = {k: v for k, v in value.items() if k != 'name'} + if _type == 'a': + yield Ip( + host=host, + ip=name, + protocol=IpProtocol.IPv4 + ) + elif _type == 'aaaa': + yield Ip( + host=host, + ip=name, + protocol=IpProtocol.IPv6 + ) + elif _type == 'ptr': + yield Subdomain( + host=name, + domain=extract_domain_info(name, domain_only=True) + ) + yield Record( + host=host, + name=name, + type=_type.upper(), + extra_data=extra_data + ) diff --git a/secator/tasks/fping.py b/secator/tasks/fping.py index abd0bc92..ade46a77 100644 --- a/secator/tasks/fping.py +++ b/secator/tasks/fping.py @@ -29,13 +29,14 @@ class fping(ReconIp): } input_type = IP output_types = [Ip] + item_loaders = [] install_cmd = 'sudo apt install -y fping' @staticmethod def item_loader(self, line): - if validators.ipv4(line) or validators.ipv6(line): - return {'ip': line, 'alive': True} - return None + if not (validators.ipv4(line) or validators.ipv6(line)): + return + yield {'ip': line, 'alive': True} @staticmethod def on_line(self, line): diff --git a/secator/tasks/gf.py b/secator/tasks/gf.py index cff04f14..591f89ff 100644 --- a/secator/tasks/gf.py +++ b/secator/tasks/gf.py @@ -23,10 +23,11 @@ class gf(Tagger): 'git clone https://github.com/1ndianl33t/Gf-Patterns $HOME/.gf || true' ) output_types = [Tag] + item_loaders = [] @staticmethod def item_loader(self, line): - return {'match': line, 'name': self.get_opt_value('pattern').rstrip() + ' pattern'} # noqa: E731,E501 + yield {'match': line, 'name': self.get_opt_value('pattern').rstrip() + ' pattern'} # noqa: E731,E501 @staticmethod def on_item(self, item): diff --git a/secator/tasks/grype.py b/secator/tasks/grype.py index 8d129167..c89466d1 100644 --- a/secator/tasks/grype.py +++ b/secator/tasks/grype.py @@ -27,6 +27,7 @@ class grype(VulnCode): USER_AGENT: OPT_NOT_SUPPORTED } output_types = [Vulnerability] + item_loaders = [] install_cmd = ( 'curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin' ) @@ -37,7 +38,7 @@ def item_loader(self, line): """Load vulnerabilty dicts from grype line output.""" split = [i for i in line.split(' ') if i] if not len(split) in [5, 6] or split[0] == 'NAME': - return None + return version_fixed = None if len(split) == 5: # no version fixed product, version, product_type, vuln_id, severity = tuple(split) @@ -76,4 +77,4 @@ def item_loader(self, line): data.update(vuln) data['severity'] = data['severity'] or severity.lower() data['extra_data'] = extra_data - return data + yield data diff --git a/secator/tasks/httpx.py b/secator/tasks/httpx.py index 04081b8f..dd2e07ed 100644 --- a/secator/tasks/httpx.py +++ b/secator/tasks/httpx.py @@ -9,8 +9,9 @@ RATE_LIMIT, RETRIES, THREADS, TIMEOUT, URL, USER_AGENT) from secator.config import CONFIG +from secator.output_types import Url, Subdomain from secator.tasks._categories import Http -from secator.utils import sanitize_url +from secator.utils import (sanitize_url, extract_domain_info, extract_subdomains_from_fqdn) @task() @@ -31,9 +32,12 @@ class httpx(Http): 'cdn': {'is_flag': True, 'default': False, 'help': 'CDN detection'}, 'debug_resp': {'is_flag': True, 'default': False, 'help': 'Debug response'}, 'vhost': {'is_flag': True, 'default': False, 'help': 'Probe and display server supporting VHOST'}, + 'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Save HTTP responses'}, # noqa: E501 'screenshot': {'is_flag': True, 'short': 'ss', 'default': False, 'help': 'Screenshot response'}, 'system_chrome': {'is_flag': True, 'default': False, 'help': 'Use local installed Chrome for screenshot'}, 'headless_options': {'is_flag': False, 'short': 'ho', 'default': None, 'help': 'Headless Chrome additional options'}, + 'follow_host_redirects': {'is_flag': True, 'short': 'fhr', 'default': None, 'help': 'Follow redirects on the same host'}, # noqa: E501 + 'tls_grab': {'is_flag': True, 'default': False, 'help': 'Grab some informations from the tls certificate'} } opt_key_map = { HEADER: 'header', @@ -55,6 +59,7 @@ class httpx(Http): THREADS: 'threads', TIMEOUT: 'timeout', USER_AGENT: OPT_NOT_SUPPORTED, + 'store_responses': 'sr', } opt_value_map = { DELAY: lambda x: str(x) + 's' if x else None, @@ -65,26 +70,54 @@ class httpx(Http): proxy_socks5 = True proxy_http = True profile = 'cpu' + output_types = [Url, Subdomain] @staticmethod def on_init(self): debug_resp = self.get_opt_value('debug_resp') if debug_resp: self.cmd = self.cmd.replace('-silent', '') - if CONFIG.http.store_responses: - self.output_response_path = f'{self.reports_folder}/response' - self.output_screenshot_path = f'{self.reports_folder}/screenshot' - os.makedirs(self.output_response_path, exist_ok=True) - os.makedirs(self.output_screenshot_path, exist_ok=True) - self.cmd += f' -sr -srd {self.reports_folder}' - - # Remove screenshot bytes and body bytes when screenshot screenshot = self.get_opt_value('screenshot') + store_responses = self.get_opt_value('store_responses') + if store_responses or screenshot: + self.cmd += f' -srd {self.reports_folder}/.outputs' if screenshot: self.cmd += ' -esb -ehb' + self.domains = [] + + @staticmethod + def on_json_loaded(self, item): + item = self._preprocess_url(item) + yield item + tls = item.get('tls', None) + if tls: + subject_cn = tls.get('subject_cn', None) + subject_an = tls.get('subject_an', []) + cert_domains = subject_an + if subject_cn: + cert_domains.append(subject_cn) + for cert_domain in cert_domains: + subdomain = self._create_subdomain_from_tls_cert(cert_domain, item['url']) + if subdomain: + yield subdomain @staticmethod - def on_item_pre_convert(self, item): + def on_end(self): + store_responses = self.get_opt_value('store_responses') + response_dir = f'{self.reports_folder}/.outputs' + if store_responses: + index_rpath = f'{response_dir}/response/index.txt' + index_spath = f'{response_dir}/screenshot/index_screenshot.txt' + index_spath2 = f'{response_dir}/screenshot/screenshot.html' + if os.path.exists(index_rpath): + os.remove(index_rpath) + if os.path.exists(index_spath): + os.remove(index_spath) + if os.path.exists(index_spath2): + os.remove(index_spath2) + + def _preprocess_url(self, item): + """Replace time string by float, sanitize URL, get final redirect URL.""" for k, v in item.items(): if k == 'time': response_time = float(''.join(ch for ch in v if not ch.isalpha())) @@ -96,10 +129,18 @@ def on_item_pre_convert(self, item): item[URL] = item.get('final_url') or item[URL] return item - @staticmethod - def on_end(self): - if CONFIG.http.store_responses: - if os.path.exists(self.output_response_path + '/index.txt'): - os.remove(self.output_response_path + '/index.txt') - if os.path.exists(self.output_screenshot_path + '/index.txt'): - os.remove(self.output_screenshot_path + '/index_screenshot.txt') + def _create_subdomain_from_tls_cert(self, domain, url): + """Extract subdomains from TLS certificate.""" + if domain.startswith('*.'): + domain = domain.lstrip('*.') + if domain in self.domains: + return None + url_domain = extract_domain_info(url) + url_domains = extract_subdomains_from_fqdn(url_domain.fqdn, url_domain.domain, url_domain.suffix) + if not url_domain or domain not in url_domains: + return None + self.domains.append(domain) + return Subdomain( + host=domain, + domain=extract_domain_info(domain, domain_only=True) + ) diff --git a/secator/tasks/katana.py b/secator/tasks/katana.py index ff13f785..c337b8ce 100644 --- a/secator/tasks/katana.py +++ b/secator/tasks/katana.py @@ -1,5 +1,4 @@ import os -import json from urllib.parse import urlparse from secator.decorators import task @@ -29,7 +28,9 @@ class katana(HttpCrawler): opts = { 'headless': {'is_flag': True, 'short': 'hl', 'help': 'Headless mode'}, 'system_chrome': {'is_flag': True, 'short': 'sc', 'help': 'Use local installed chrome browser'}, - 'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'} + 'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'}, + 'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Store responses'}, # noqa: E501 + 'form_fill': {'type': bool, 'short': 'ff', 'help': 'Enable form filling'} } opt_key_map = { HEADER: 'headers', @@ -50,7 +51,9 @@ class katana(HttpCrawler): RETRIES: 'retry', THREADS: 'concurrency', TIMEOUT: 'timeout', - USER_AGENT: OPT_NOT_SUPPORTED + USER_AGENT: OPT_NOT_SUPPORTED, + 'store_responses': 'sr', + 'form_fill': 'aff' } opt_value_map = { DELAY: lambda x: int(x) if isinstance(x, float) else x @@ -70,7 +73,6 @@ class katana(HttpCrawler): # TAGS: lambda x: x['response'].get('server') } } - item_loaders = [] install_cmd = 'sudo apt install build-essential && go install -v github.com/projectdiscovery/katana/cmd/katana@latest' install_github_handle = 'projectdiscovery/katana' proxychains = False @@ -79,12 +81,7 @@ class katana(HttpCrawler): profile = 'io' @staticmethod - def item_loader(self, item): - try: - item = json.loads(item) - except json.JSONDecodeError: - return None - + def on_json_loaded(self, item): # form detection forms = item.get('response', {}).get('forms', []) if forms: @@ -107,14 +104,16 @@ def on_init(self): debug_resp = self.get_opt_value('debug_resp') if debug_resp: self.cmd = self.cmd.replace('-silent', '') - if CONFIG.http.store_responses: - self.cmd += f' -sr -srd {self.reports_folder}' + store_responses = self.get_opt_value('store_responses') + if store_responses: + self.cmd += f' -srd {self.reports_folder}/.outputs' @staticmethod def on_item(self, item): if not isinstance(item, Url): return item - if CONFIG.http.store_responses and os.path.exists(item.stored_response_path): + store_responses = self.get_opt_value('store_responses') + if store_responses and os.path.exists(item.stored_response_path): with open(item.stored_response_path, 'r', encoding='latin-1') as fin: data = fin.read().splitlines(True) first_line = data[0] @@ -126,5 +125,7 @@ def on_item(self, item): @staticmethod def on_end(self): - if CONFIG.http.store_responses and os.path.exists(self.reports_folder + '/index.txt'): - os.remove(self.reports_folder + '/index.txt') + store_responses = self.get_opt_value('store_responses') + index_rpath = f'{self.reports_folder}/.outputs/index.txt' + if store_responses and os.path.exists(index_rpath): + os.remove(index_rpath) diff --git a/secator/tasks/mapcidr.py b/secator/tasks/mapcidr.py index 50261290..e8a57b5f 100644 --- a/secator/tasks/mapcidr.py +++ b/secator/tasks/mapcidr.py @@ -17,6 +17,7 @@ class mapcidr(ReconIp): install_github_handle = 'projectdiscovery/mapcidr' input_type = CIDR_RANGE output_types = [Ip] + item_loaders = [] opt_key_map = { THREADS: OPT_NOT_SUPPORTED, PROXY: OPT_NOT_SUPPORTED, @@ -29,5 +30,5 @@ class mapcidr(ReconIp): @staticmethod def item_loader(self, line): if validators.ipv4(line) or validators.ipv6(line): - return {'ip': line, 'alive': False} - return None + yield {'ip': line, 'alive': False} + return diff --git a/secator/tasks/naabu.py b/secator/tasks/naabu.py index 398e6ea1..21eb826f 100644 --- a/secator/tasks/naabu.py +++ b/secator/tasks/naabu.py @@ -14,7 +14,7 @@ class naabu(ReconPort): file_flag = '-list' json_flag = '-json' opts = { - PORTS: {'type': str, 'short': 'p', 'help': 'Ports'}, + PORTS: {'type': str, 'short': 'p', 'help': 'Ports (default: nmap\'s top 100 ports'}, TOP_PORTS: {'type': str, 'short': 'tp', 'help': 'Top ports'}, 'scan_type': {'type': str, 'help': 'Scan type (SYN (s)/CONNECT(c))'}, # 'health_check': {'is_flag': True, 'short': 'hc', 'help': 'Health check'} diff --git a/secator/tasks/nmap.py b/secator/tasks/nmap.py index c23c3162..025a0891 100644 --- a/secator/tasks/nmap.py +++ b/secator/tasks/nmap.py @@ -8,7 +8,7 @@ from secator.decorators import task from secator.definitions import (CONFIDENCE, CVSS_SCORE, DELAY, DESCRIPTION, EXTRA_DATA, FOLLOW_REDIRECT, - HEADER, HOST, ID, IP, MATCHED_AT, NAME, + HEADER, HOST, ID, IP, PROTOCOL, MATCHED_AT, NAME, OPT_NOT_SUPPORTED, OUTPUT_PATH, PORT, PORTS, PROVIDER, PROXY, RATE_LIMIT, REFERENCE, REFERENCES, RETRIES, SCRIPT, SERVICE_NAME, SEVERITY, STATE, TAGS, @@ -24,18 +24,21 @@ @task() class nmap(VulnMulti): """Network Mapper is a free and open source utility for network discovery and security auditing.""" - cmd = 'nmap -sT -sV -Pn' + cmd = 'nmap' input_flag = None input_chunk_size = 1 file_flag = '-iL' opt_prefix = '--' output_types = [Port, Vulnerability, Exploit] opts = { - PORTS: {'type': str, 'short': 'p', 'help': 'Ports to scan'}, + PORTS: {'type': str, 'short': 'p', 'help': 'Ports to scan (default: most common 1000 ports for each protocol)'}, TOP_PORTS: {'type': int, 'short': 'tp', 'help': 'Top ports to scan [full, 100, 1000]'}, SCRIPT: {'type': str, 'default': 'vulners', 'help': 'NSE scripts'}, - # 'tcp_connect': {'type': bool, 'short': 'sT', 'default': False, 'help': 'TCP Connect scan'}, + 'skip_host_discovery': {'is_flag': True, 'short': 'Pn', 'default': False, 'help': 'Skip host discovery (no ping)'}, + 'version_detection': {'is_flag': True, 'short': 'sV', 'default': False, 'help': 'Version detection'}, 'tcp_syn_stealth': {'is_flag': True, 'short': 'sS', 'default': False, 'help': 'TCP SYN Stealth'}, + 'tcp_connect': {'is_flag': True, 'short': 'sT', 'default': False, 'help': 'TCP Connect scan'}, + 'udp_scan': {'is_flag': True, 'short': 'sU', 'default': False, 'help': 'UDP scan'}, 'output_path': {'type': str, 'short': 'oX', 'default': None, 'help': 'Output XML file path'}, } opt_key_map = { @@ -51,7 +54,12 @@ class nmap(VulnMulti): # Nmap opts PORTS: '-p', - 'output_path': '-oX' + 'skip_host_discovery': '-Pn', + 'version_detection': '-sV', + 'tcp_connect': '-sT', + 'tcp_syn_stealth': '-sS', + 'udp_scan': '-sU', + 'output_path': '-oX', } opt_value_map = { PORTS: lambda x: ','.join([str(p) for p in x]) if isinstance(x, list) else x @@ -73,6 +81,15 @@ def on_init(self): output_path = f'{self.reports_folder}/.outputs/{self.unique_name}.xml' self.output_path = output_path self.cmd += f' -oX {self.output_path}' + tcp_syn_stealth = self.get_opt_value('tcp_syn_stealth') + tcp_connect = self.get_opt_value('tcp_connect') + if tcp_syn_stealth: + self.cmd = f'sudo {self.cmd}' + if tcp_connect and tcp_syn_stealth: + self._print( + 'Options -sT (SYN stealth scan) and -sS (CONNECT scan) are conflicting. Keeping only -sT.', + 'bold gold3') + self.cmd = self.cmd.replace('-sT ', '') def yielder(self): yield from super().yielder() @@ -118,7 +135,7 @@ def __iter__(self): # Get extra data extra_data = self._get_extra_data(port) - service_name = extra_data['service_name'] + service_name = extra_data.get('service_name', '') version_exact = extra_data.get('version_exact', False) conf = extra_data.get('confidence') if not version_exact: @@ -133,6 +150,9 @@ def __iter__(self): # Get script output scripts = self._get_scripts(port) + # Get port protocol + protocol = port['@protocol'].upper() + # Yield port data port = { PORT: port_number, @@ -140,6 +160,7 @@ def __iter__(self): STATE: state, SERVICE_NAME: service_name, IP: ip, + PROTOCOL: protocol, EXTRA_DATA: extra_data } yield port @@ -200,11 +221,19 @@ def _get_hostname(self, host_cfg): if hostnames: hostname = hostnames[0]['@name'] else: - hostname = host_cfg.get('address', {}).get('@addr', None) + hostname = self._get_address(host_cfg).get('@addr', None) return hostname + def _get_address(self, host_cfg): + if isinstance(host_cfg.get('address', {}), list): + addresses = host_cfg.get('address', {}) + for address in addresses: + if address.get('@addrtype') == "ipv4": + return address + return host_cfg.get('address', {}) + def _get_ip(self, host_cfg): - return host_cfg.get('address', {}).get('@addr', None) + return self._get_address(host_cfg).get('@addr', None) def _get_extra_data(self, port_cfg): extra_data = { diff --git a/secator/tasks/searchsploit.py b/secator/tasks/searchsploit.py index e8b21406..912861d1 100644 --- a/secator/tasks/searchsploit.py +++ b/secator/tasks/searchsploit.py @@ -1,3 +1,5 @@ +import re + from secator.decorators import task from secator.definitions import (CVES, EXTRA_DATA, ID, MATCHED_AT, NAME, PROVIDER, REFERENCE, TAGS, OPT_NOT_SUPPORTED) @@ -5,6 +7,9 @@ from secator.runners import Command +SEARCHSPLOIT_TITLE_REGEX = re.compile(r'^((?:[a-zA-Z\-_!\.()]+\d?\s?)+)\.?\s*(.*)$') + + @task() class searchsploit(Command): """Exploit-DB command line search tool.""" @@ -19,12 +24,15 @@ class searchsploit(Command): output_types = [Exploit] output_map = { Exploit: { - NAME: lambda x: '-'.join(x['Title'].split('-')[1:]).strip(), - PROVIDER: lambda x: 'EDB', + NAME: 'Title', ID: 'EDB-ID', + PROVIDER: lambda x: 'EDB', CVES: lambda x: [c for c in x['Codes'].split(';') if c.startswith('CVE-')], REFERENCE: lambda x: f'https://exploit-db.com/exploits/{x["EDB-ID"]}', - EXTRA_DATA: lambda x: {'verified': x['Verified']} + TAGS: lambda x: searchsploit.tags_extractor(x), + EXTRA_DATA: lambda x: { + k.lower().replace('date_', ''): v for k, v in x.items() if k not in ['Title', 'EDB-ID', 'Codes', 'Tags', 'Source'] and v != '' # noqa: E501 + } } } install_cmd = 'sudo git clone https://gitlab.com/exploit-database/exploitdb.git /opt/exploitdb || true && sudo ln -sf /opt/exploitdb/searchsploit /usr/local/bin/searchsploit' # noqa: E501 @@ -34,6 +42,18 @@ class searchsploit(Command): input_chunk_size = 1 profile = 'io' + @staticmethod + def tags_extractor(item): + tags = [] + for tag in item['Tags'].split(','): + _tag = '_'.join( + tag.lower().replace('-', '_',).replace('(', '').replace(')', '').split(' ') + ) + if not _tag: + continue + tags.append(tag) + return tags + @staticmethod def before_init(self): _in = self.input @@ -49,5 +69,23 @@ def before_init(self): def on_item_pre_convert(self, item): if self.matched_at: item[MATCHED_AT] = self.matched_at - item[TAGS] = [self.input.replace('\'', '')] + return item + + @staticmethod + def on_item(self, item): + match = SEARCHSPLOIT_TITLE_REGEX.match(item.name) + # if not match: + # self._print(f'[bold red]{item.name} ({item.reference}) did not match SEARCHSPLOIT_TITLE_REGEX. Please report this issue.[/]') # noqa: E501 + if match: + group = match.groups() + product = '-'.join(group[0].strip().split(' ')) + if len(group[1]) > 1: + versions, title = tuple(group[1].split(' - ')) + item.name = title + product_info = [f'{product.lower()} {v.strip()}' for v in versions.split('/')] + item.tags = product_info + item.tags + # else: + # self._print(f'[bold red]{item.name} ({item.reference}) did not quite match SEARCHSPLOIT_TITLE_REGEX. Please report this issue.[/]') # noqa: E501 + input_tag = '-'.join(self.input.replace('\'', '').split(' ')) + item.tags = [input_tag] + item.tags return item diff --git a/secator/utils.py b/secator/utils.py index abb9df12..51c75543 100644 --- a/secator/utils.py +++ b/secator/utils.py @@ -4,12 +4,14 @@ import logging import operator import os +import tldextract import re import select import sys +import validators import warnings -from datetime import datetime +from datetime import datetime from inspect import isclass from pathlib import Path from pkgutil import iter_modules @@ -262,8 +264,8 @@ def merge_opts(*options): all_opts = {} for opts in options: if opts: - opts_noemtpy = {k: v for k, v in opts.items() if v is not None} - all_opts.update(opts_noemtpy) + opts_noempty = {k: v for k, v in opts.items() if v is not None} + all_opts.update(opts_noempty) return all_opts @@ -343,8 +345,10 @@ def print_results_table(results, title=None, exclude_fields=[], log=False): if output_type.__name__ == 'Progress': continue items = [ - item for item in results if item._type == output_type.get_name() and not item._duplicate + item for item in results if item._type == output_type.get_name() ] + if CONFIG.runners.remove_duplicates: + items = [item for item in items if not item._duplicate] if items: _table = build_table( items, @@ -437,3 +441,54 @@ def print_version(): console.print(f'[bold gold3]Lib folder[/]: {LIB_FOLDER}') if status == 'outdated': console.print('[bold red]secator is outdated, run "secator update" to install the latest version.') + + +def extract_domain_info(input, domain_only=False): + """Extracts domain info from a given any URL or FQDN. + + Args: + input (str): An URL or FQDN. + + Returns: + tldextract.ExtractResult: Extracted info. + str | None: Registered domain name or None if invalid domain (only if domain_only is set). + """ + result = tldextract.extract(input) + if not result or not result.domain or not result.suffix: + return None + if domain_only: + if not validators.domain(result.registered_domain): + return None + return result.registered_domain + return result + + +def extract_subdomains_from_fqdn(fqdn, domain, suffix): + """ + Generates a list of subdomains up to the root domain from a fully qualified domain name (FQDN). + + Args: + fqdn (str): The full domain name, e.g., 'console.cloud.google.com'. + domain (str): The main domain, e.g., 'google'. + suffix (str): The top-level domain (TLD), e.g., 'com'. + + Returns: + List[str]: A list containing the FQDN and all its subdomains down to the root domain. + """ + # Start with the full domain and prepare to break it down + parts = fqdn.split('.') + + # Initialize the list of subdomains with the full domain + subdomains = [fqdn] + + # Continue stripping subdomains until reaching the base domain (domain + suffix) + base_domain = f"{domain}.{suffix}" + current = fqdn + + while current != base_domain: + # Remove the leftmost part of the domain + parts = parts[1:] + current = '.'.join(parts) + subdomains.append(current) + + return subdomains diff --git a/secator/utils_test.py b/secator/utils_test.py index 27c0ef2c..712b1bef 100644 --- a/secator/utils_test.py +++ b/secator/utils_test.py @@ -88,8 +88,12 @@ USER_AGENT: 'Mozilla/5.0 (Windows NT 5.1; rv:7.0.1) Gecko/20100101 Firefox/7.0.1', # Individual tasks options + 'bup.mode': 'http_methods', 'gf.pattern': 'xss', 'nmap.output_path': load_fixture('nmap_output', FIXTURES_DIR, only_path=True, ext='.xml'), # nmap XML fixture + 'nmap.tcp_connect': True, + 'nmap.version_detection': True, + 'nmap.skip_host_discovery': True, 'msfconsole.resource': load_fixture('msfconsole_input', FIXTURES_DIR, only_path=True), 'dirsearch.output_path': load_fixture('dirsearch_output', FIXTURES_DIR, only_path=True), 'maigret.output_path': load_fixture('maigret_output', FIXTURES_DIR, only_path=True), diff --git a/tests/fixtures/bup_output.json b/tests/fixtures/bup_output.json new file mode 100644 index 00000000..354b77c3 --- /dev/null +++ b/tests/fixtures/bup_output.json @@ -0,0 +1,16 @@ +{ + "request_curl_cmd": "/usr/bin/curl -sS -kgi -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36' --path-as-is -X TRACE https://example.com", + "request_curl_payload": "-X TRACE https://example.com", + "response_headers": "HTTP/1.1 200 OK\nDate: Fri, 03 May 2024 14:24:50 GMT\nServer: Apache/2.4.56 (Debian) mod_fcgid/2.3.9 OpenSSL/1.1.1w\nConnection: close\nTransfer-Encoding: chunked\nContent-Type: message/http", + "response_data": "TRACE / HTTP/1.1\nHost: example.com\nAccept: */*\nUser-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36", + "response_status_code": 200, + "response_content_type": "message/http", + "response_content_length": "", + "response_lines_count": 3, + "response_words_count": 14, + "response_title": "", + "response_server_type": "Apache/2.4.56 (Debian) mod_fcgid/2.3.9 OpenSSL/1.1.1w", + "response_redirect_url": "", + "response_html_filename": "bypass-81890dda67871d1d84165e46c6ab89fc.html", + "request_url": "https://example.com" +} \ No newline at end of file diff --git a/tests/integration/inputs.py b/tests/integration/inputs.py index 097adbf5..ad8892ed 100644 --- a/tests/integration/inputs.py +++ b/tests/integration/inputs.py @@ -6,6 +6,7 @@ USERNAME: 'ocervell', IP: '127.0.0.1', CIDR_RANGE: '192.168.1.0/24', + 'bup': 'http://localhost:3000/ftp/coupons_2013.md.bak', 'dalfox': 'http://testphp.vulnweb.com/listproducts.php?cat=123&artist=123&asdf=ff', 'ffuf': 'http://localhost:3000/FUZZ', 'gf': 'http://localhost:3000?q=test', diff --git a/tests/integration/outputs.py b/tests/integration/outputs.py index 4826b818..1b79a64c 100644 --- a/tests/integration/outputs.py +++ b/tests/integration/outputs.py @@ -3,6 +3,22 @@ Vulnerability, Record) OUTPUTS_TASKS = { + 'bup': [ + Url( + url='http://localhost:3000/ftp/coupons_2013.md.bak', + status_code=403, + content_length=164, + content_type='text/html', + method='GET', + _source='bup' + ), + Url( + url='http://localhost:3000/ftp/coupons_2013.md.bak', + status_code=405, + method='SEARCH', + _source='bup' + ) + ], 'cariddi': [ Url( url='http://localhost:3000/robots.txt', @@ -59,7 +75,7 @@ _source='dnsx' ), Record( - name='v=spf1 include:wikimedia.org ~all', + name='v=spf1 include:_cidrs.wikimedia.org ~all', type='TXT', host='wikipedia.org', _source='dnsx' @@ -217,7 +233,7 @@ ], 'wpscan': [ Tag( - name='Wordpress theme - twentytwentyfour 1.1', + name='Wordpress theme - twentytwentyfour 1.2', match='http://localhost:8000/', _source='wpscan'), Vulnerability( diff --git a/tests/integration/test_tasks.py b/tests/integration/test_tasks.py index e5a531b3..fcc6982a 100644 --- a/tests/integration/test_tasks.py +++ b/tests/integration/test_tasks.py @@ -49,6 +49,9 @@ def test_tasks(self): 'feroxbuster.filter_size': 1987, 'h8mail.local_breach': load_fixture('h8mail_breach', INTEGRATION_DIR, only_path=True), 'nmap.port': '3000,8080', + 'nmap.tcp_connect': True, + 'nmap.version_detection': True, + 'nmap.skip_host_discovery': True, 'match_codes': '200', 'maigret.site': 'github', 'wordlist': load_fixture('wordlist', INTEGRATION_DIR, only_path=True), diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 00000000..e4b09820 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,40 @@ +import unittest +from secator.utils import extract_domain_info + +class TestExtractRootDomain(unittest.TestCase): + def test_root_domain_extraction(self): + domains = [ + ("subdomain.example.com", "example.com"), + ("www.subdomain.example.co.uk", "example.co.uk"), + ("example.com", "example.com"), + ("ex-ample.co", "ex-ample.co"), + ("test--domain.com", 'test--domain.com'), + ("-example.com", None), + ("example-.com", None), + ("exa_mple.com", None), + ("exa--mple.com", 'exa--mple.com'), + ("example.longtld", None), + ("", None), + ("localhost", None), + ("192.168.1.1", None), + ("test.domain-.com", None), + ("test.-domain.com", None), + ("test_domain.com", None), + ("sub.domain_goes.com", None), + ("okay.domain.gov", "domain.gov"), + # Adding Unicode domain examples + ("täst.example.org", "example.org"), # Normal IDN + ("münchen.de", "münchen.de"), # City domain name in German + ("пример.рф", "пример.рф"), # Example in Cyrillic + ("中文网.中国", "中文网.中国"), # Chinese characters + ("xn--fiq228c5hs.xn--fiq64b", "xn--fiq228c5hs.xn--fiq64b"), # Punycode representation of Chinese domain + ("test.みんな", "test.みんな"), # Using Japanese TLD + ("http://sub.domain.пример.рф", "пример.рф"), + ("https://suрф.みんな.пример.рф", "пример.рф"), + ("http://mydomain.localhost", None), + ] + + for domain, expected in domains: + with self.subTest(domain=domain): + result = extract_domain_info(domain, domain_only=True) + self.assertEqual(result, expected, f"Failed for domain: {domain}")