Take advantage of LMDB append optimization (#1912)

* Take advantage of LMDB append optimization

* Assert return values of put/putmulti

* Remove an inappropriate use of append

synapse/lib/lmdbslab.py

@@ -367,7 +367,8 @@ async def puts(self, name, items, reqid=None):
 
         for item in items:
 
-            self.slab.put(abrv + s_common.int64en(offs), s_msgpack.en(item), db=self.qdata)
+            putv = self.slab.put(abrv + s_common.int64en(offs), s_msgpack.en(item), db=self.qdata)
+            assert putv, 'Put failed'
 
             self.sizes.inc(name, 1)
             offs = self.offsets.inc(name, 1)
@@ -1421,9 +1422,9 @@ def pop(self, lkey, db=None):
     def delete(self, lkey, val=None, db=None):
         return self._xact_action(self.delete, lmdb.Transaction.delete, lkey, val, db=db)
 
-    def put(self, lkey, lval, dupdata=False, overwrite=True, db=None):
+    def put(self, lkey, lval, dupdata=False, overwrite=True, append=False, db=None):
         return self._xact_action(self.put, lmdb.Transaction.put, lkey, lval, dupdata=dupdata, overwrite=overwrite,
-                                 db=db)
+                                 append=append, db=db)
 
     def replace(self, lkey, lval, db=None):
         '''

synapse/lib/slabseqn.py

@@ -40,7 +40,8 @@ def add(self, item, indx=None):
             return indx
 
         indx = self.indx
-        self.slab.put(s_common.int64en(indx), s_msgpack.en(item), db=self.db)
+        retn = self.slab.put(s_common.int64en(indx), s_msgpack.en(item), append=True, db=self.db)
+        assert retn, "Not adding the largest index"
 
         self.indx += 1
 
@@ -89,9 +90,11 @@ def save(self, items):
 
             rows.append((lkey, byts))
 
-        self.slab.putmulti(rows, append=True, db=self.db)
+        retn = self.slab.putmulti(rows, append=True, db=self.db)
         took = s_common.now() - tick
 
+        assert retn, "Not adding the largest indices"
+
         origindx = self.indx
         self.indx = indx
 

synapse/tests/test_lib_lmdbslab.py

@@ -124,7 +124,9 @@ async def test_lmdbslab_base(self):
 
             slab.put(b'\x00\x01', b'hehe', db=baz)
             slab.put(b'\xff', b'haha', db=baz)
-            slab.put(b'\xff\xff', b'hoho', db=baz)
+
+            slab.put(b'\xff\xff', b'hoho', append=True, db=baz)  # Should succeed
+            slab.put(b'\xaa\xff', b'hoho', append=True, db=baz)  # Should fail (not the last key)
 
             self.true(slab.dirty)
 
@@ -151,6 +153,7 @@ async def test_lmdbslab_base(self):
             self.false(slab.prefexists(b'\x02', db=baz))
             self.true(slab.prefexists(b'\xff\xff', db=baz))
             self.false(slab.prefexists(b'\xff\xff', db=foo))
+            self.false(slab.prefexists(b'\xaa\xff', db=baz))
 
             self.true(slab.rangeexists(b'\x00', b'\x01', db=baz))
             self.true(slab.rangeexists(b'\x00\x00', b'\x00\x04', db=baz))