Merge pull request #1 from CDJellen/VIRTS-3055-@2329-Issue_csv_fix

Fix csv file format in operation report download.

README.md

@@ -58,16 +58,16 @@ These requirements are for the computer running the core framework:
 
 Concise installation steps:
 ```Bash
-git clone https://github.com/mitre/caldera.git --recursive --branch 4.0.0-alpha
+git clone https://github.com/mitre/caldera.git --recursive
 cd caldera
 pip3 install -r requirements.txt
 python3 server.py --insecure
 ```
 
 Full steps:
-Start by cloning this repository recursively, passing the desired version/release in x.x.x format. This will pull in all available plugins. If you clone master - or any non-release branch - you may experience bugs.
+Start by cloning this repository recursively, passing the desired version/release in x.x.x format. This will pull in all available plugins.
 ```Bash
-git clone https://github.com/mitre/caldera.git --recursive --branch 4.0.0-alpha
+git clone https://github.com/mitre/caldera.git --recursive --branch x.x.x
 ```
 
 Next, install the PIP requirements:

app/api/packs/advanced.py

@@ -22,8 +22,6 @@ async def enable(self):
         self.app_svc.application.router.add_route('GET', '/advanced/configurations', self._section_configurations)
         self.app_svc.application.router.add_route('GET', '/advanced/exfills', self._section_exfil_files)
 
-    """ PRIVATE """
-
     @check_authorization
     @template('planners.html')
     async def _section_planners(self, request):

app/api/packs/campaign.py

@@ -21,8 +21,6 @@ async def enable(self):
         self.app_svc.application.router.add_route('GET', '/campaign/profiles', self._section_profiles)
         self.app_svc.application.router.add_route('GET', '/campaign/operations', self._section_operations)
 
-    """ PRIVATE """
-
     @check_authorization
     @template('agents.html')
     async def _section_agent(self, request):
@@ -110,8 +108,6 @@ def load_usage_markdown(header):
         return dict(operations=operations, groups=groups, adversaries=adversaries, sources=sources, planners=planners,
                     obfuscators=obfuscators, usage=usage)
 
-    """ PRIVATE """
-
     @staticmethod
     def _rollup_abilities(abilities):
         rolled = defaultdict(list)

app/api/rest_api.py

@@ -43,8 +43,6 @@ async def enable(self):
         self.app_svc.application.router.add_route('GET', '/api/{index}', self.rest_core_info)
         self.app_svc.application.router.add_route('GET', '/file/download_exfil', self.download_exfil_file)
 
-    """ BOILERPLATE """
-
     @template('login.html', status=401)
     async def login(self, request):
         return dict()
@@ -65,8 +63,6 @@ async def landing(self, request):
         data = dict(plugins=[p.display for p in plugins], errors=self.app_svc.errors + self._request_errors(request))
         return render_template('%s.html' % access[0].name, request, data)
 
-    """ API ENDPOINTS """
-
     @check_authorization
     async def rest_core(self, request):
         try:
@@ -165,8 +161,6 @@ def is_in_exfil_dir(f):
                 return web.HTTPNotFound(body=str(e))
         return web.HTTPBadRequest(body='A file needs to be specified for download')
 
-    """ PRIVATE """
-
     @staticmethod
     def _request_errors(request):
         errors = []

app/api/v2/handlers/base_object_api.py

@@ -23,8 +23,6 @@ def __init__(self, description, obj_class, schema, ram_key, id_property, auth_sv
     def add_routes(self, app: web.Application):
         pass
 
-    """GET"""
-
     async def get_all_objects(self, request: web.Request):
         access = await self.get_request_permissions(request)
 
@@ -48,8 +46,6 @@ async def get_object(self, request: web.Request):
 
         return self._api_manager.dump_object_with_filters(obj, include, exclude)
 
-    """POST"""
-
     async def create_object(self, request: web.Request):
         data = await request.json()
 
@@ -75,8 +71,6 @@ async def _error_if_object_with_id_exists(self, obj_id: str):
             if self._api_manager.find_object(self.ram_key, search):
                 raise JsonHttpBadRequest(f'{self.description.capitalize()} with given id already exists: {obj_id}')
 
-    """PATCH"""
-
     async def update_object(self, request: web.Request):
         data, access, obj_id, query, search = await self._parse_common_data_from_request(request)
 
@@ -94,8 +88,6 @@ async def update_on_disk_object(self, request: web.Request):
             raise JsonHttpNotFound(f'{self.description.capitalize()} not found: {obj_id}')
         return obj
 
-    """PUT"""
-
     async def create_or_update_object(self, request: web.Request):
         data, access, obj_id, query, search = await self._parse_common_data_from_request(request)
 
@@ -120,8 +112,6 @@ async def create_or_update_on_disk_object(self, request: web.Request):
 
         return obj
 
-    """DELETE"""
-
     async def delete_object(self, request: web.Request):
         obj_id = request.match_info.get(self.id_property)
 
@@ -141,8 +131,6 @@ async def delete_on_disk_object(self, request: web.Request):
         obj_id = request.match_info.get(self.id_property)
         await self._api_manager.remove_object_from_disk_by_id(identifier=obj_id, ram_key=self.ram_key)
 
-    """Helpers"""
-
     async def _parse_common_data_from_request(self, request) -> (dict, dict, str, dict, dict):
         data = {}
         raw_body = await request.read()

app/api/v2/handlers/operation_api.py

@@ -163,8 +163,6 @@ async def get_potential_links_by_paw(self, request: web.Request):
         potential_links = await self._api_manager.get_potential_links(operation_id, access, paw)
         return web.json_response(potential_links)
 
-    '''Overridden Methods'''
-
     async def create_object(self, request: web.Request):
         data = await request.json()
         await self._error_if_object_with_id_exists(data.get(self.id_property))
@@ -178,8 +176,6 @@ async def update_object(self, request: web.Request):
             raise JsonHttpNotFound(f'{self.description.capitalize()} not found: {obj_id}')
         return obj
 
-    ''' PRIVATE '''
-
     async def _read_output_parameter_(self, request: web.Request):
         raw_body = await request.read()
         output = False

app/api/v2/managers/ability_api_manager.py

@@ -45,7 +45,6 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro
         await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access)
         return next(self.find_objects(ram_key, {id_property: obj_id}))
 
-    '''Helpers'''
     def _validate_ability_data(self, create: bool, data: dict):
         # Correct ability_id key for ability file saving.
         data['id'] = data.pop('ability_id', '')

app/api/v2/managers/base_api_manager.py

@@ -23,8 +23,6 @@ def __init__(self, data_svc, file_svc, logger=None):
     def log(self):
         return self._log
 
-    """Object Retrieval"""
-
     def find_objects(self, ram_key: str, search: dict = None):
         """Find objects matching the given criteria"""
         for obj in self._data_svc.ram[ram_key]:
@@ -54,8 +52,6 @@ def dump_object_with_filters(obj: Any, include: List[str] = None, exclude: List[
                 dumped.pop(exclude_attribute, None)
         return dumped
 
-    """Object Creation"""
-
     def create_object_from_schema(self, schema: SchemaMeta, data: dict, access: BaseWorld.Access):
         obj_schema = schema()
         obj = obj_schema.load(data)
@@ -79,8 +75,6 @@ def _get_allowed_from_access(self, access) -> BaseWorld.Access:
         else:
             return self._data_svc.Access.RED
 
-    """Object Updates"""
-
     def find_and_update_object(self, ram_key: str, data: dict, search: dict = None):
         for obj in self.find_objects(ram_key, search):
             new_obj = self.update_object(obj, data)
@@ -112,17 +106,13 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro
         await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access)
         return next(self.find_objects(ram_key, {id_property: obj_id}))
 
-    """"Object Replacement"""
-
     async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str):
         obj_id = getattr(obj, id_property)
         file_path = await self._get_existing_object_file_path(obj_id, ram_key)
 
         await self._save_and_reload_object(file_path, data, type(obj), obj.access)
         return next(self.find_objects(ram_key, {id_property: obj_id}))
 
-    """"Object Removal"""
-
     async def remove_object_from_memory_by_id(self, identifier: str, ram_key: str, id_property: str):
         await self._data_svc.remove(ram_key, {id_property: identifier})
 
@@ -132,8 +122,6 @@ async def remove_object_from_disk_by_id(self, identifier: str, ram_key: str):
         if os.path.exists(file_path):
             os.remove(file_path)
 
-    """Helpers"""
-
     @staticmethod
     async def _get_new_object_file_path(identifier: str, ram_key: str) -> str:
         """Create file path for new object"""

app/api/v2/managers/operation_api_manager.py

@@ -122,7 +122,6 @@ async def get_potential_links(self, operation_id: str, access: dict, paw: str =
         potential_links = [potential_link.display for potential_link in operation.potential_links]
         return potential_links
 
-    """Object Creation Helpers"""
     async def get_operation_object(self, operation_id: str, access: dict):
         try:
             operation = (await self._data_svc.locate('operations', {'id': operation_id}))[0]

app/contacts/contact_gist.py

@@ -141,8 +141,6 @@ async def get_uploads(self):
             self.log.error('Receiving file uploads over c2 (%s) failed: %s' % (self.__class__.__name__, e))
             return []
 
-    """ PRIVATE """
-
     async def _send_instructions(self, agent, instructions):
         response = dict(paw=agent.paw,
                         sleep=await agent.calculate_sleep(),

app/contacts/contact_html.py

@@ -16,8 +16,6 @@ def __init__(self, services):
     async def start(self):
         self.app_svc.application.router.add_route('*', self.get_config('app.contact.html'), self._accept_beacon)
 
-    """ PRIVATE """
-
     @template('weather.html')
     async def _accept_beacon(self, request):
         try:

app/contacts/contact_http.py

@@ -17,8 +17,6 @@ def __init__(self, services):
     async def start(self):
         self.app_svc.application.router.add_route('POST', '/beacon', self._beacon)
 
-    """ PRIVATE """
-
     async def _beacon(self, request):
         try:
             profile = json.loads(self.contact_svc.decode_bytes(await request.read()))

app/contacts/contact_slack.py

@@ -150,8 +150,6 @@ async def get_uploads(self):
             self.log.error('Receiving file uploads over c2 (%s) failed: %s' % (self.__class__.__name__, e))
             return []
 
-    """ PRIVATE """
-
     async def _send_instructions(self, agent, instructions):
         response = dict(paw=agent.paw,
                         sleep=await agent.calculate_sleep(),

app/contacts/contact_tcp.py

@@ -93,8 +93,6 @@ async def send(self, session_id: int, cmd: str, timeout: int = 60) -> Tuple[int,
             self.log.exception(e)
             return 1, '~$ ', str(e), ''
 
-    """ PRIVATE """
-
     @staticmethod
     async def _handshake(reader):
         profile_bites = (await reader.readline()).strip()

app/objects/c_ability.py

@@ -1,5 +1,4 @@
 import collections
-import os
 import uuid
 
 import marshmallow as ma
@@ -8,7 +7,6 @@
 from app.objects.secondclass.c_executor import ExecutorSchema
 from app.objects.secondclass.c_requirement import RequirementSchema
 from app.utility.base_object import BaseObject
-from app.utility.base_service import BaseService
 from app.utility.base_world import AccessSchema
 
 
@@ -27,6 +25,7 @@ class AbilitySchema(ma.Schema):
     additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String())
     access = ma.fields.Nested(AccessSchema, missing=None)
     singleton = ma.fields.Bool(missing=None)
+    plugin = ma.fields.String(missing=None)
 
     @ma.pre_load
     def fix_id(self, data, **_):
@@ -58,7 +57,7 @@ def executors(self):
 
     def __init__(self, ability_id='', name=None, description=None, tactic=None, technique_id=None, technique_name=None,
                  executors=(), requirements=None, privilege=None, repeatable=False, buckets=None, access=None,
-                 additional_info=None, tags=None, singleton=False, **kwargs):
+                 additional_info=None, tags=None, singleton=False, plugin='', **kwargs):
         super().__init__()
         self.ability_id = ability_id if ability_id else str(uuid.uuid4())
         self.tactic = tactic.lower() if tactic else None
@@ -80,6 +79,7 @@ def __init__(self, ability_id='', name=None, description=None, tactic=None, tech
         self.additional_info = additional_info or dict()
         self.additional_info.update(**kwargs)
         self.tags = set(tags) if tags else set()
+        self.plugin = plugin
 
     def __getattr__(self, item):
         try:
@@ -103,14 +103,11 @@ def store(self, ram):
         existing.update('buckets', self.buckets)
         existing.update('tags', self.tags)
         existing.update('singleton', self.singleton)
+        existing.update('plugin', self.plugin)
         return existing
 
     async def which_plugin(self):
-        file_svc = BaseService.get_service('file_svc')
-        for plugin in os.listdir('plugins'):
-            if await file_svc.walk_file_path(os.path.join('plugins', plugin, 'data', ''), '%s.yml' % self.ability_id):
-                return plugin
-        return None
+        return self.plugin
 
     def find_executor(self, name, platform):
         return self._executor_map.get(self._make_executor_map_key(name, platform))

app/objects/c_adversary.py

@@ -1,11 +1,9 @@
-import os
 import uuid
 
 import marshmallow as ma
 
 from app.objects.interfaces.i_object import FirstClassObjectInterface
 from app.utility.base_object import BaseObject
-from app.utility.base_service import BaseService
 
 
 DEFAULT_OBJECTIVE_ID = '495a9828-cab1-44dd-a0ca-66e58177d8cc'
@@ -20,6 +18,7 @@ class AdversarySchema(ma.Schema):
     objective = ma.fields.String()
     tags = ma.fields.List(ma.fields.String(), allow_none=True)
     has_repeatable_abilities = ma.fields.Boolean(dump_only=True)
+    plugin = ma.fields.String(missing=None)
 
     @ma.pre_load
     def fix_id(self, adversary, **_):
@@ -57,7 +56,7 @@ class Adversary(FirstClassObjectInterface, BaseObject):
     def unique(self):
         return self.hash('%s' % self.adversary_id)
 
-    def __init__(self, name='', adversary_id='', description='', atomic_ordering=(), objective='', tags=None):
+    def __init__(self, name='', adversary_id='', description='', atomic_ordering=(), objective='', tags=None, plugin=''):
         super().__init__()
         self.adversary_id = adversary_id if adversary_id else str(uuid.uuid4())
         self.name = name
@@ -66,6 +65,7 @@ def __init__(self, name='', adversary_id='', description='', atomic_ordering=(),
         self.objective = objective or DEFAULT_OBJECTIVE_ID
         self.tags = set(tags) if tags else set()
         self.has_repeatable_abilities = False
+        self.plugin = plugin
 
     def store(self, ram):
         existing = self.retrieve(ram['adversaries'], self.unique)
@@ -78,6 +78,7 @@ def store(self, ram):
         existing.update('objective', self.objective)
         existing.update('tags', self.tags)
         existing.update('has_repeatable_abilities', self.check_repeatable_abilities(ram['abilities']))
+        existing.update('plugin', self.plugin)
         return existing
 
     def verify(self, log, abilities, objectives):
@@ -101,11 +102,7 @@ def has_ability(self, ability):
         return False
 
     async def which_plugin(self):
-        file_svc = BaseService.get_service('file_svc')
-        for plugin in os.listdir('plugins'):
-            if await file_svc.walk_file_path(os.path.join('plugins', plugin, 'data', ''), '%s.yml' % self.adversary_id):
-                return plugin
-        return None
+        return self.plugin
 
     def check_repeatable_abilities(self, ability_list):
         return any(ab.repeatable for ab_id in self.atomic_ordering for ab in ability_list if ab.ability_id == ab_id)

app/objects/c_agent.py

@@ -332,8 +332,6 @@ def assign_pending_executor_change(self):
         self._executor_change_to_assign = None
         return executor_change
 
-    """ PRIVATE """
-
     def _replace_payload_data(self, decoded_cmd, file_svc):
         for uuid in re.findall(self.RESERVED['payload'], decoded_cmd):
             if self.is_uuid4(uuid):

app/objects/c_operation.py

@@ -337,8 +337,6 @@ async def write_event_logs_to_disk(self, file_svc, data_svc, output=False):
         await self._write_logs_to_disk(event_logs, file_name, event_logs_dir, file_svc)
         logging.debug('Wrote event logs for operation %s to disk at %s/%s' % (self.name, event_logs_dir, file_name))
 
-    """ PRIVATE """
-
     async def _write_logs_to_disk(self, logs, file_name, dest_dir, file_svc):
         logs_dumps = json.dumps(logs)
         await file_svc.save_file(file_name, logs_dumps.encode(), dest_dir, encrypt=False)