Dockerized kanku-sho, now it is more portable. Also fixed a bug in th…

…e interface about HTTP headers, and updated README.md.

.gitignore

@@ -9,4 +9,7 @@ __pycache__
 src/extract_dumps.sh
 src/config.JSON
 src/Analysis_done/
-!src/Analysis_done/.gitkeep
+!src/Analysis_done/.gitkeep
+volume
+!volume/traffic_analysed/.gitkeep
+!volume/traffic_queue/.gitkeep

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN echo "wireshark-common wireshark-common/install-setuid boolean false" | debconf-set-selections
+RUN apt-get update -y && apt-get install -y --force-yes rsync sqlite3 tcpdump tshark
+
+COPY . .
+
+CMD ["./docker_wrapper.sh"]

README.md

@@ -12,6 +12,8 @@ kanku-sho is a tool for the following workflow:
 <br />
 The remote machine should have SSH enabled, and in order for the remote sniffing to work properly, you have to install an identity file on the remote machine.
 <br />
+If you want, you can also capture local traffic, by configuring "capture" -> "local_capture" -> true in config.JSON (see more later about which config.JSON you should use); in this case, "remote_pcap_folder" will actually be a local folder.
+<br />
 The task is easy:
 <br />
 ssh-keygen -f your_path/your_key -t ecdsa -b 521
@@ -29,7 +31,21 @@ The main modules are three:
 
 3) flask_interface.py
 <br />
-(1) and (2) can be executed together by run_threads.py, but depending on your operating system, there could be issues with pyshark.
+
+# Run with Docker
+<br /> <br />
+It is the easiest setup. In this case, you don't have to edit the config.JSON that you find in /src folder, but the config.JSON that you find in /volume folder.
+<br />
+The best thing to do is to put your identity file in the volume, to avoid re-building the image every time. Specify its path in config.JSON, under "global" -> "identity_file".
+<br />
+Since you may want to execute just the interface, for example, you can specify in "run" dict of config.JSON which modules you want to run (set true if you want to run the module).
+<br /> <br />
+
+# Run on the host
+<br /> <br />
+A little bit harder setup but more performant (Docker adds a layer to a pipeline).
+<br /> <br />
+Modules (1) and (2) can be executed together by run_threads.py, but depending on your operating system, there could be issues with pyshark.
 <br /> <br />
 Before using a new database, you have to run init_database.py, which will create a new database with the pre-defined DB schema, with the name defined in config.JSON. The database should be first created with: <br />
 sqlite3 db_name <br />

docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+    kanku-sho:
+        build:
+            context: .
+            dockerfile: ./Dockerfile
+            network: host
+        image: kanku-sho
+        volumes:
+            - ./volume/:/usr/src/app/volume/
+        ports:
+            - "8000:8000" 

docker_wrapper.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# This is intended to run the main modules in Docker, because Docker allows only one CMD at a time
+srcPath=/usr/src/app/src
+vol=/usr/src/app/volume
+config="$vol/config.JSON"
+python -u "$srcPath/init_database.py" "$config" > "$vol/init_database_log.txt"
+declare -a pids
+nohup python -u "$srcPath/flask_interface.py" "$config" > "$vol/flask_interface_log.txt" 2>&1 &
+pids[0]=$!
+nohup python -u "$srcPath/remote_sniffer.py" "$config" > "$vol/remote_sniffer_log.txt" 2>&1 &
+pids[1]=$!
+nohup python -u "$srcPath/analysis_controller.py" "$config" > "$vol/analysis_controller_log.txt" 2>&1 &
+pids[2]=$!
+for pid in ${pids[*]}; do
+  wait $pid
+done

src/analysis_controller.py

@@ -56,6 +56,10 @@ def greedy_analysis(global_config, capture_config, traffic_db):
         config_name = sys.argv[1]
     with open(config_name, 'r') as f:
         config = json.load(f)
+    if 'run' in config.keys():
+        if not config['run']['analysis_controller']:
+            print("analysis_controller.py: exiting because of run configuration.")
+            exit(0)
     global_config = config['global']
     capture_config = config['capture']
     traffic_db = config['traffic_db']

src/config.JSON

@@ -18,10 +18,11 @@
     "num_circular_files": 500,
     "initial_sleep_time": 10,
     "capture_file_size": 1,
-    "preserve_all": false
+    "preserve_all": false,
+    "local_capture": false
   },
   "traffic_db": {
-    "db_name": "/home/marcofelix98/CyberChallenge/attack_defense/mHackeroni/mhackeroni_with_headers.db"
+    "db_name": "/home/marcofelix98/CyberChallenge/attack_defense/final.db"
   },
   "flask": {
     "port": 8000,

src/flask_interface.py

@@ -1,6 +1,7 @@
 from flask import Flask, render_template, request, Response
 import json
 import html
+import sys
 from interface import get_capture_files, get_streams, get_stream_with_payloads, get_services_stats, \
     get_possible_http_exploits, get_possible_tcp_exploits
 from util_code_generation import generate_from_stream
@@ -10,9 +11,8 @@
 
 app = Flask(__name__)
 
-with open('config.JSON', 'r') as f:
-    config = json.load(f)
-db_name = config['traffic_db']['db_name']
+global config
+global db_name
 
 
 @app.route("/")
@@ -120,5 +120,18 @@ def show_tcp_exploits():
 
 
 if __name__ == "__main__":
+    global db_name
+    global config
+    if len(sys.argv) < 2:
+        config_name = 'config.JSON'
+    else:
+        config_name = sys.argv[1]
+    with open(config_name, 'r') as f:
+        config = json.load(f)
+    if 'run' in config.keys():
+        if not config['run']['flask_interface']:
+            print("flask_interface.py: exiting because of run configuration.")
+            exit(0)
+    db_name = config['traffic_db']['db_name']
     app.debug = config['flask']['debug']
     app.run(host=config['flask']['host'], port=config['flask']['port'])

src/init_database.py

@@ -1,10 +1,21 @@
 import json
+import sys
+import os
 
 from sqlite_functions import init_database
 
 
 if __name__ == "__main__":
-    with open('config.JSON', 'r') as f:
+    if len(sys.argv) < 2:
+        config_name = 'config.JSON'
+    else:
+        config_name = sys.argv[1]
+    with open(config_name, 'r') as f:
         config = json.load(f)
-    traffic_db = config['traffic_db']
-    init_database(traffic_db['db_name'])
+    traffic_db = config['traffic_db']['db_name']
+    db_name = traffic_db.split("/")[-1]
+    traffic_dir = os.listdir("/".join(traffic_db.split("/")[:-1]))
+    if db_name not in traffic_dir:
+        init_database(traffic_db)
+    else:
+        print("init_database.py: Database already initialized.")

src/remote_sniffer.py

@@ -1,8 +1,9 @@
 import json
 import sys
 import time
+import os
 
-from remote_wrapper import *
+from remote_wrapper import ssh_wrapper, remote_getfile_wrapper
 
 
 def capture_command(port, interface, filename, run_with_sudo, num_files, file_size):
@@ -26,22 +27,30 @@ def remote_sniffing(global_config, capture_config):
     num_files = capture_config['num_circular_files']
     init_sleep_time = capture_config['initial_sleep_time']
     cap_file_size = capture_config['capture_file_size']
+    is_local_capture = capture_config['local_capture']
     port = global_config['port']
     sudo = "sudo " if run_with_sudo else ""
     sleep_time = init_sleep_time
     filename = "{}_{}.pcap".format(prefix, interface)
     remote_file_path = remote_path + filename
     cap_comm = capture_command(port, interface, remote_file_path, run_with_sudo, num_files, cap_file_size)
-    ssh_wrapper(global_config, cap_comm)
+    if not is_local_capture:
+        ssh_wrapper(global_config, cap_comm)
+    else:
+        os.system(cap_comm)
     while True:
         print("Sleep time: {}".format(sleep_time))
         time.sleep(sleep_time)
         start = time.time()
-        remote_getfile_wrapper(global_config, remote_path + "*.bz2", local_path)
+        if not is_local_capture:
+            remote_getfile_wrapper(global_config, remote_path + "*.bz2", local_path)
+        else:
+            os.system("mv {}*.bz2 {}".format(remote_path, local_path))
         end = time.time()
         sleep_time = min(1 + init_sleep_time / (end-start), init_sleep_time)
         os.system('bzip2 -df {}*.bz2'.format(local_path))
-        ssh_wrapper(global_config, sudo + "rm -f {}".format(remote_file_path + "*.bz2"))
+        if not is_local_capture:
+            ssh_wrapper(global_config, sudo + "rm -f {}".format(remote_file_path + "*.bz2"))
 
 
 if __name__ == "__main__":
@@ -51,6 +60,10 @@ def remote_sniffing(global_config, capture_config):
         config_name = sys.argv[1]
     with open(config_name, 'r') as f:
         config = json.load(f)
+    if 'run' in config.keys():
+        if not config['run']['remote_sniffer']:
+            print("remote_sniffer.py: exiting because of run configuration.")
+            exit(0)
     global_config = config['global']
     capture_config = config['capture']
     remote_sniffing(global_config, capture_config)

src/remote_wrapper.py

@@ -6,7 +6,8 @@ def ssh_wrapper(global_config, remote_command):
     host = global_config['host']
     port = global_config['port']
     identity_file = global_config['identity_file']
-    command = "ssh -i {} -p {} {}@{} '{}'".format(identity_file, port, user, host, remote_command)
+    command = "ssh -oStrictHostKeyChecking=no -i {} -p {} {}@{} '{}'".format(identity_file, port,
+                                                                             user, host, remote_command)
     os.system(command)
 
 
@@ -17,7 +18,8 @@ def remote_getfile_wrapper(global_config, remote_file_path, local_path):
     identity_file = global_config['identity_file']
     if local_path[0] != '/':
         local_path = '/' + local_path
-    command = 'rsync -avz -e "ssh -i {} -p {}" {}@{}:{} {}'.format(identity_file, port,
+    command = 'rsync -avz -e "ssh -oStrictHostKeyChecking=no -i {} -p {}" {}@{}:{} {}'.format(
+                                                                    identity_file, port,
                                                                    user, host, remote_file_path,
                                                                    local_path)
     os.system(command)

src/templates/show_stream_http.html

@@ -38,7 +38,7 @@ <h1><center>Stream with id {{ stream['id'] }} (n. {{ stream['number'] }} of pcap
                             <td>
                                 <div class="popup" onclick="clickFunction(event, {{ 4 * payload['id'] }})">Show parameters
                                       <span class="popuptext" id="{{ 4 * payload['id'] }}">
-                                        {{ payload['data'] }}
+                                        {{ payload['http']['parameters'] }}
                                       </span>
                                 </div>
                             </td>
@@ -70,14 +70,14 @@ <h1><center>Stream with id {{ stream['id'] }} (n. {{ stream['number'] }} of pcap
                         <td>
                             <div class="popup" onclick="clickFunction(event, {{ 4 * payload['id'] + 2 }})">Show payload data
                                   <span class="popuptext" id="{{ 4 * payload['id'] + 2 }}">
-                                    {{ payload['data'].split('\r\n\r\n')[1] }}
+                                    {{ payload['data'].split('\r\n\r\n')[-1] }}
                                   </span>
                             </div>
                         </td>
                         <td>
                             <div class="popup" onclick="clickFunction(event, {{ 4 * payload['id'] + 3 }})">Show headers
                                   <span class="popuptext" id="{{ 4 * payload['id'] + 3 }}">
-                                    {{ payload['data'].split('\r\n\r\n')[0] }}
+                                    {{ payload['data'].split('\r\n\r\n')[-2] }}
                                   </span>
                             </div>
                         </td>

volume/config.JSON

@@ -0,0 +1,37 @@
+{
+  "run": {
+    "flask_interface": true,
+    "remote_sniffer": false,
+    "analysis_controller": false
+  },
+  "global": {
+    "flag_regex": "flg\\{[A-Za-z0-9-_]{1,}\\}",
+    "round_timeout": 60,
+    "user": "mininet",
+    "host": "192.168.122.115",
+    "port": "22",
+    "identity_file": "/usr/src/app/volume/remote_VM_ssh_id"
+  },
+  "capture": {
+    "remote_interface": "any",
+    "remote_pcap_folder": "/home/mininet",
+    "local_pcap_folder": "/usr/src/app/volume/traffic_queue",
+    "local_pcap_backup": "/usr/src/app/volume/traffic_analysed",
+    "cap_filename_prefix": "remote_VM",
+    "run_with_sudo": false,
+    "time_string": "%d_%m_%Y_%H:%M",
+    "num_circular_files": 500,
+    "initial_sleep_time": 10,
+    "capture_file_size": 1,
+    "preserve_all": false,
+    "local_capture": false
+  },
+  "traffic_db": {
+    "db_name": "/usr/src/app/volume/traffic.db"
+  },
+  "flask": {
+    "port": 8000,
+    "host": "0.0.0.0",
+    "debug": true
+  }
+}