Axon HTTPAPI tweaks

docs/synapse/devguides/devops_axon.rst

@@ -32,23 +32,31 @@ The Axon application implements the Synapse Cell class and as such can be config
 implementations. For details on the general configuration options see :ref:`devops-cell-config`.
 For a list of boot time configuration options for the Axon, see the listing at :ref:`autodoc-axon-conf`.
 
-The Axon application utilizes local storage,
-and has similar system requirements as other object storage systems (without data replication).
+The Axon application utilizes local storage, and has similar system requirements as other object storage systems
+(without data replication).
 
 When deployed as a remote server it is a best practice to add a new service user,
 which can be accomplished with ``synapse.tools.cellauth``. ::
 
     python -m synapse.tools.cellauth tcp://root:<root_passwd>@<svc_ip>:<svc_port> modify svcuser1 --adduser
     python -m synapse.tools.cellauth tcp://root:<root_passwd>@<svc_ip>:<svc_port> modify svcuser1 --passwd secret
 
+Generally, most individual end users do not need to have discrete Axon accounts; instead the Axon's users are generally
+service users, through which other systems interact with the Axon to store and retrieve files.
+
+CLI Tools
+---------
+
+For CLI tools related to uploading and downloading files from an Axon, see :ref:`syn-tools-pullfile` and
+:ref:`syn-tools-pushfile` documentation.
+
 Backups
 -------
 
 It is strongly recommended that users schedule regular backups of the Axon.
 
-When the Axon is deployed as a client within another application
-(e.g. if no remote Axon Telepath URL is specified in a Cortex)
-it will be backed up as part of the parent application backup process on the top-level directory.
+When the Axon is deployed as a client within another application (e.g. if no remote Axon Telepath URL is specified in
+a Cortex) it will be backed up as part of the parent application backup process on the top-level directory.
 
 If the Axon is deployed as a standalone application, the Synapse backup tool can be used on the directory
 specified during service startup.

docs/synapse/devguides/devops_cortex.rst

@@ -71,9 +71,8 @@ Configuring A Remote Axon
 By default a Cortex will initialize a local :ref:`gloss-axon` for general object / blob storage, which will also
 be shared as ``axon`` on the same Telepath URL as the Cortex.
 
-However, an Axon can instead be deployed as a remote server (see :ref:`devops-axon`)
-and configured in the Cortex by specifying the Axon's Telepath URL
-in the ``axon`` configuration parameter (see :ref:`autodoc-cortex-conf`).
+However, an Axon can instead be deployed as a remote server (see :ref:`devops-axon`) and configured in the Cortex by
+specifying the Axon's Telepath URL in the ``axon`` configuration parameter (see :ref:`autodoc-cortex-conf`).
 
 For example, if the remote Axon is listening on port ``27592``, and has a service user ``core00``, then the
 Cortex ``cell.yaml`` file could have the following configuration::
@@ -82,6 +81,9 @@ Cortex ``cell.yaml`` file could have the following configuration::
     axon: tcp://core00:secret@<axon_host_ip>:27592
     ...
 
+For interacting with byte storage inside of Storm, see :ref:`stormlibs-lib-bytes` for APIs related to interacting with
+the Axon.
+
 .. _200_migration:
 
 0.1.x to 2.x.x Migration

docs/synapse/httpapi.ipynb

@@ -448,7 +448,25 @@
     "    *Returns*\n",
     "        On successful upload, or if the file already existed, the API returns information about the file::\n",
     "        \n",
-    "            (size, SHA-256)\n",
+    "            {\n",
+    "              \"md5\": \"<the md5sum value of the uploaded bytes>\",\n",
+    "              \"sha1\": \"<the sha1 value of the uploaded bytes>\",\n",
+    "              \"sha256\": \"<the sha256 value of the uploaded bytes>\",\n",
+    "              \"sha512\": \"<the sha512 value of the uploaded bytes>\",\n",
+    "              \"size\": <the size of the uploaded bytes>\n",
+    "            }\n",
+    "\n",
+    "\n",
+    "/api/v1/axon/files/has/sha256/<SHA-256>\n",
+    "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
+    "\n",
+    "This API allows the caller to check if a file exists in the Axon as identified by the SHA-256.\n",
+    "\n",
+    "*Method*\n",
+    "    GET\n",
+    "    \n",
+    "    *Returns*\n",
+    "        True if the file exists; False if the file does not exist.\n",
     "\n",
     "\n",
     "/api/v1/axon/files/by/sha256/<SHA-256>\n",
@@ -481,7 +499,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.8"
+   "version": "3.7.3"
   }
  },
  "nbformat": 4,

synapse/axon.py

@@ -10,6 +10,7 @@
 import synapse.lib.base as s_base
 import synapse.lib.const as s_const
 import synapse.lib.share as s_share
+import synapse.lib.hashset as s_hashset
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.slabseqn as s_slabseqn
@@ -20,7 +21,7 @@
 MAX_SPOOL_SIZE = CHUNK_SIZE * 32  # 512 mebibytes
 MAX_HTTP_UPLOAD_SIZE = 4 * s_const.tebibyte
 
-class AxonFilesHttpV1(s_httpapi.StreamHandler):
+class AxonHttpUploadV1(s_httpapi.StreamHandler):
 
     async def prepare(self):
         self.upfd = None
@@ -32,10 +33,12 @@ async def prepare(self):
         self.request.connection.set_max_body_size(MAX_HTTP_UPLOAD_SIZE)
 
         self.upfd = await self.cell.upload()
+        self.hashset = s_hashset.HashSet()
 
     async def data_received(self, chunk):
         if chunk is not None:
             await self.upfd.write(chunk)
+            self.hashset.update(chunk)
             await asyncio.sleep(0)
 
     def on_finish(self):
@@ -47,11 +50,15 @@ def on_connection_close(self):
 
     async def _save(self):
         size, sha256b = await self.upfd.save()
-        sha256 = s_common.ehex(sha256b)
 
-        self.sendRestRetn((size, sha256))
+        fhashes = {htyp: hasher.hexdigest() for htyp, hasher in self.hashset.hashes}
 
-        return
+        assert sha256b == s_common.uhex(fhashes.get('sha256'))
+        assert size == self.hashset.size
+
+        fhashes['size'] = size
+
+        return self.sendRestRetn(fhashes)
 
     async def post(self):
         '''
@@ -64,7 +71,17 @@ async def put(self):
         await self._save()
         return
 
-class AxonFileHttpV1(s_httpapi.Handler):
+
+class AxonHttpHasV1(s_httpapi.Handler):
+
+    async def get(self, sha256):
+        if not await self.reqAuthAllowed(('axon', 'has')):
+            return
+        resp = await self.cell.has(s_common.uhex(sha256))
+        return self.sendRestRetn(resp)
+
+
+class AxonHttpDownloadV1(s_httpapi.Handler):
 
     async def get(self, sha256):
 
@@ -239,8 +256,9 @@ async def _initBlobStor(self):
         self.onfini(self.blobslab.fini)
 
     def _initAxonHttpApi(self):
-        self.addHttpApi('/api/v1/axon/files/put', AxonFilesHttpV1, {'cell': self})
-        self.addHttpApi('/api/v1/axon/files/by/sha256/([0-9a-fA-F]{64}$)', AxonFileHttpV1, {'cell': self})
+        self.addHttpApi('/api/v1/axon/files/put', AxonHttpUploadV1, {'cell': self})
+        self.addHttpApi('/api/v1/axon/files/has/sha256/([0-9a-fA-F]{64}$)', AxonHttpHasV1, {'cell': self})
+        self.addHttpApi('/api/v1/axon/files/by/sha256/([0-9a-fA-F]{64}$)', AxonHttpDownloadV1, {'cell': self})
 
     def _addSyncItem(self, item):
         self.axonhist.add(item)

synapse/tests/test_axon.py

@@ -194,6 +194,7 @@ async def test_axon_http(self):
             await newb.setPasswd('secret')
 
             url_ul = f'https://localhost:{port}/api/v1/axon/files/put'
+            url_hs = f'https://localhost:{port}/api/v1/axon/files/has/sha256'
             url_dl = f'https://localhost:{port}/api/v1/axon/files/by/sha256'
 
             asdfhash_h = s_common.ehex(asdfhash)
@@ -207,6 +208,11 @@ async def test_axon_http(self):
                     item = await resp.json()
                     self.eq('err', item.get('status'))
 
+                async with sess.get(f'{url_hs}/{asdfhash_h}') as resp:
+                    self.eq(403, resp.status)
+                    item = await resp.json()
+                    self.eq('err', item.get('status'))
+
                 async with sess.post(url_ul, data=abuf) as resp:
                     self.eq(403, resp.status)
                     item = await resp.json()
@@ -220,6 +226,7 @@ async def test_axon_http(self):
                         pass
 
             await newb.addRule((True, ('axon', 'get')))
+            await newb.addRule((True, ('axon', 'has')))
             await newb.addRule((True, ('axon', 'upload')))
 
             # Basic
@@ -232,18 +239,35 @@ async def test_axon_http(self):
                     item = await resp.json()
                     self.eq('err', item.get('status'))
 
+                async with sess.get(f'{url_hs}/{asdfhash_h}') as resp:
+                    self.eq(200, resp.status)
+                    item = await resp.json()
+                    self.eq('ok', item.get('status'))
+                    self.false(item.get('result'))
+
                 async with sess.post(url_ul, data=abuf) as resp:
                     self.eq(200, resp.status)
                     item = await resp.json()
                     self.eq('ok', item.get('status'))
-                    self.eq((asdfretn[0], asdfhash_h), item.get('result'))
+                    result = item.get('result')
+                    self.eq(set(result.keys()), {'size', 'md5', 'sha1', 'sha256', 'sha512'})
+                    self.eq(result.get('size'), asdfretn[0])
+                    self.eq(result.get('sha256'), asdfhash_h)
                     self.true(await axon.has(asdfhash))
 
+                async with sess.get(f'{url_hs}/{asdfhash_h}') as resp:
+                    self.eq(200, resp.status)
+                    item = await resp.json()
+                    self.eq('ok', item.get('status'))
+                    self.true(item.get('result'))
+
                 async with sess.put(url_ul, data=abuf) as resp:
                     self.eq(200, resp.status)
                     item = await resp.json()
                     self.eq('ok', item.get('status'))
-                    self.eq((asdfretn[0], asdfhash_h), item.get('result'))
+                    result = item.get('result')
+                    self.eq(result.get('size'), asdfretn[0])
+                    self.eq(result.get('sha256'), asdfhash_h)
                     self.true(await axon.has(asdfhash))
 
                 async with sess.get(f'{url_dl}/{asdfhash_h}') as resp:
@@ -257,7 +281,9 @@ async def test_axon_http(self):
                     self.eq(200, resp.status)
                     item = await resp.json()
                     self.eq('ok', item.get('status'))
-                    self.eq((bbufretn[0], bbufhash_h), item.get('result'))
+                    result = item.get('result')
+                    self.eq(result.get('size'), bbufretn[0])
+                    self.eq(result.get('sha256'), bbufhash_h)
                     self.true(await axon.has(bbufhash))
 
                 byts = io.BytesIO(bbuf)
@@ -266,7 +292,9 @@ async def test_axon_http(self):
                     self.eq(200, resp.status)
                     item = await resp.json()
                     self.eq('ok', item.get('status'))
-                    self.eq((bbufretn[0], bbufhash_h), item.get('result'))
+                    result = item.get('result')
+                    self.eq(result.get('size'), bbufretn[0])
+                    self.eq(result.get('sha256'), bbufhash_h)
                     self.true(await axon.has(bbufhash))
 
                 byts = io.BytesIO(b'')
@@ -275,7 +303,9 @@ async def test_axon_http(self):
                     self.eq(200, resp.status)
                     item = await resp.json()
                     self.eq('ok', item.get('status'))
-                    self.eq((emptyretn[0], emptyhash_h), item.get('result'))
+                    result = item.get('result')
+                    self.eq(result.get('size'), emptyretn[0])
+                    self.eq(result.get('sha256'), emptyhash_h)
                     self.true(await axon.has(emptyhash))
 
                 # Streaming download