New feature: implement locking via Lock API

README.md

@@ -10,7 +10,14 @@ This service is part of the ScienceMesh Interoperability Platform (IOP) and impl
 It enables ScienceMesh EFSS storages to integrate Office Online platforms including Microsoft Office Online and Collabora Online. In addition it implements a [bridge](src/bridge/readme.md) module with dedicated extensions to support apps like CodiMD and Etherpad.
 
 Author: Giuseppe Lo Presti (@glpatcern) <br/>
-Contributions: Michael DSilva (@madsi1m), Lovisa Lugnegaard (@LovisaLugnegard), Samuel Alfageme (@SamuAlfageme), Ishank Arora (@ishank011), Willy Kloucek (@wkloucek)
+Contributors:
+- Michael DSilva (@madsi1m)
+- Lovisa Lugnegaard (@LovisaLugnegard)
+- Samuel Alfageme (@SamuAlfageme)
+- Ishank Arora (@ishank011)
+- Willy Kloucek (@wkloucek)
+- Gianmaria Del Monte (@gmgigi96)
+- Klaas Freitag (@dragotin)
 
 Initial revision: December 2016 <br/>
 First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))<br/>

requirements.txt

@@ -8,5 +8,5 @@ requests
 more_itertools
 tuspy
 prometheus-flask-exporter
-cs3apis>=0.1.dev66
+cs3apis>=0.1.dev87
 waitress

src/bridge/__init__.py

@@ -1,7 +1,7 @@
 '''
-The WOPI Bridge for IOP. This connector service supports CodiMD and Etherpad.
+The WOPI bridge extension for IOP. This connector service supports CodiMD and Etherpad.
 
-Author: [email protected], CERN/IT-ST
+Main author: [email protected], CERN/IT-ST
 '''
 
 import os
@@ -20,6 +20,7 @@
 from base64 import urlsafe_b64encode
 import flask
 import bridge.wopiclient as wopic
+import core.wopiutils as utils
 
 
 # The supported plugins integrated with this WOPI Bridge
@@ -171,6 +172,7 @@ def appopen(wopisrc, acctok):
                                          'lastsave': int(time.time()) - WB.saveinterval,
                                          'toclose': {acctok[-20:]: False},
                                          'docid': wopilock['docid'],
+                                         'app': os.path.splitext(filemd['BaseFileName'])[1][1:],
                                         }
             # also clear any potential stale response for this document
             try:
@@ -282,15 +284,15 @@ def run(self):
                         self.cleanup(openfile, wopisrc, wopilock)
                     except Exception as e:    # pylint: disable=broad-except
                         ex_type, ex_value, ex_traceback = sys.exc_info()
-                        WB.log.error('msg="SaveThread: unexpected exception caught" ex="%s" type="%s" traceback="%s"' %
-                                     (e, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback)))
-        WB.log.info('msg="SaveThread terminated, shutting down"')
+                        WB.log.critical('msg="SaveThread: unexpected exception caught" ex="%s" type="%s" traceback="%s"' %
+                                        (e, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback)))
 
     def savedirty(self, openfile, wopisrc):
         '''save documents that are dirty for more than `saveinterval` or that are being closed'''
         wopilock = None
         if openfile['tosave'] and (_intersection(openfile['toclose'])
                                    or (openfile['lastsave'] < time.time() - WB.saveinterval)):
+            app = BRIDGE_EXT_PLUGINS.get(openfile['app']) if 'app' in openfile else None
             try:
                 wopilock = wopic.getlock(wopisrc, openfile['acctok'])
             except wopic.InvalidLock:
@@ -301,14 +303,23 @@ def savedirty(self, openfile, wopisrc):
                         wopisrc, openfile['acctok'], openfile['docid'], _intersection(openfile['toclose']))
                 except wopic.InvalidLock as ile:
                     # even this attempt failed, give up
-                    # TODO here we should save the file on a local storage to help later recovery
                     WB.saveresponses[wopisrc] = wopic.jsonify(str(ile)), http.client.INTERNAL_SERVER_ERROR
+                    # attempt to save to local storage to help for later recovery: this is a feature of the core wopiserver
+                    content = rc = None
+                    if app:
+                        content, rc = WB.plugins[app].savetostorage(wopisrc, openfile['acctok'], False, {'docid': openfile['docid']}, onlyfetch=True)
+                        if rc == http.client.OK:
+                            utils.storeForRecovery(content, wopisrc[wopisrc.rfind('/')+1:], openfile['acctok'][-20:], ile)
+                    if rc != http.client.OK:
+                        WB.log.error('msg="SaveThread: failed to fetch file for recovery to local storage" token="%s" docid="%s" app="%s" response="%s"' %
+                                     (openfile['acctok'][-20:], openfile['docid'], app, content))
                     # set some 'fake' metadata, will be automatically cleaned up later
                     openfile['lastsave'] = int(time.time())
                     openfile['tosave'] = False
                     openfile['toclose'] = {'invalid-lock': True}
                     return None
-            app = BRIDGE_EXT_PLUGINS.get(wopilock['app'])
+            if not app:
+                app = BRIDGE_EXT_PLUGINS.get(wopilock['app'])
             if not app:
                 WB.log.error('msg="SaveThread: malformed app attribute in WOPI lock" lock="%s"' % wopilock)
                 WB.saveresponses[wopisrc] = wopic.jsonify('Unrecognized app for this file'), http.client.BAD_REQUEST
@@ -369,14 +380,17 @@ def cleanup(self, openfile, wopisrc, wopilock):
                     del WB.openfiles[wopisrc]
             elif openfile['toclose'] != wopilock['toclose']:
                 # some user still on it, refresh lock if the toclose part has changed
-                wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose'])
+                try:
+                    wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose'])
+                except wopic.InvalidLock:
+                    WB.log.warning('msg="SaveThread: failed to refresh lock, will try again later" url="%s"' % wopisrc)
 
 
 @atexit.register
 def stopsavethread():
     '''Exit handler to cleanly stop the storage sync thread'''
     if WB.savethread:
-        WB.log.info('msg="Waiting for SaveThread to complete"')
+        WB.log.info('msg="Waiting for SaveThread to complete"')  # TODO when this handler is called, the logger is not accessible any longer
         with WB.savecv:
             WB.active = False
             WB.savecv.notify()

src/bridge/codimd.py

@@ -3,7 +3,7 @@
 
 The CodiMD-specific code used by the WOPI bridge.
 
-Author: [email protected], CERN/IT-ST
+Main author: [email protected], CERN/IT-ST
 '''
 
 import os
@@ -233,13 +233,16 @@ def _getattachments(mddoc, docfilename, forcezip=False):
     return zip_buffer.getvalue(), response
 
 
-def savetostorage(wopisrc, acctok, isclose, wopilock):
+def savetostorage(wopisrc, acctok, isclose, wopilock, onlyfetch=False):
     '''Copy document from CodiMD back to storage'''
     # get document from CodiMD
     try:
         log.info('msg="Fetching file from CodiMD" isclose="%s" appurl="%s" token="%s"' %
                  (isclose, appurl + wopilock['docid'], acctok[-20:]))
         mddoc = _fetchfromcodimd(wopilock, acctok)
+        if onlyfetch:
+            # this is used only in case of recovery to local storage
+            return mddoc, http.client.OK
     except AppFailure:
         return wopic.jsonify('Could not save file, failed to fetch document from CodiMD'), http.client.INTERNAL_SERVER_ERROR
 
@@ -269,11 +272,15 @@ def savetostorage(wopisrc, acctok, isclose, wopilock):
         if isclose and wopilock['digest'] == 'dirty':
             h = hashlib.sha1()
             h.update(mddoc)
-        wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest=(h.hexdigest() if h else 'dirty'))
-        log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' %
-                 (wopilock['filename'], isclose, acctok[-20:]))
-        # combine the responses
-        return attresponse if attresponse else (wopic.jsonify('File saved successfully'), http.client.OK)
+        try:
+            wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest=(h.hexdigest() if h else 'dirty'))
+            log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' %
+                     (wopilock['filename'], isclose, acctok[-20:]))
+            # combine the responses
+            return attresponse if attresponse else (wopic.jsonify('File saved successfully'), http.client.OK)
+        except wopic.InvalidLock:
+            return wopic.jsonify('File saved, but failed to refresh lock'), http.client.INTERNAL_SERVER_ERROR
+
 
     # on close, use saveas for either the new bundle, if this is the first time we have attachments,
     # or the single md file, if there are no more attachments.

src/bridge/etherpad.py

@@ -3,7 +3,7 @@
 
 The Etherpad-specific code used by the WOPI bridge.
 
-Author: [email protected], CERN/IT-ST
+Main author: [email protected], CERN/IT-ST
 '''
 
 from random import choice
@@ -134,13 +134,16 @@ def _fetchfrometherpad(wopilock, acctok):
         raise AppFailure
 
 
-def savetostorage(wopisrc, acctok, isclose, wopilock):
+def savetostorage(wopisrc, acctok, isclose, wopilock, onlyfetch=False):
     '''Copy document from Etherpad back to storage'''
     # get document from Etherpad
     try:
         log.info('msg="Fetching file from Etherpad" isclose="%s" appurl="%s" token="%s"' %
                  (isclose, appurl + '/p' + wopilock['docid'], acctok[-20:]))
         epfile = _fetchfrometherpad(wopilock, acctok)
+        if onlyfetch:
+            # this is used only in case of recovery to local storage
+            return epfile, http.client.OK
     except AppFailure:
         return wopic.jsonify('Could not save file, failed to fetch document from Etherpad'), http.client.INTERNAL_SERVER_ERROR
 
@@ -158,7 +161,10 @@ def savetostorage(wopisrc, acctok, isclose, wopilock):
     reply = wopic.handleputfile('PutFile', wopisrc, res)
     if reply:
         return reply
-    wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest='dirty')
-    log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' %
-             (wopilock['filename'], isclose, acctok[-20:]))
-    return wopic.jsonify('File saved successfully'), http.client.OK
+    try:
+        wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest='dirty')
+        log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' %
+                (wopilock['filename'], isclose, acctok[-20:]))
+        return wopic.jsonify('File saved successfully'), http.client.OK
+    except wopic.InvalidLock:
+        return wopic.jsonify('File saved, but failed to refresh lock'), http.client.INTERNAL_SERVER_ERROR

src/bridge/wopiclient.py

@@ -1,9 +1,9 @@
 '''
 wopiclient.py
 
-A set of WOPI functions for the WOPI bridge service.
+A set of WOPI client functions for the WOPI bridge service.
 
-Author: [email protected], CERN/IT-ST
+Main author: [email protected], CERN/IT-ST
 '''
 
 import os
@@ -113,7 +113,7 @@ def refreshlock(wopisrc, acctok, wopilock, digest=None, toclose=None):
         # else fail
     log.error('msg="Calling WOPI RefreshLock failed" url="%s" response="%d" reason="%s"' %
               (wopisrc, res.status_code, res.headers.get('X-WOPI-LockFailureReason')))
-    return None
+    raise InvalidLock('Failed to refresh the lock')
 
 
 def relock(wopisrc, acctok, docid, isclose):
@@ -154,8 +154,8 @@ def handleputfile(wopicall, wopisrc, res):
         return jsonify('Error saving the file. %s' % res.headers.get('X-WOPI-LockFailureReason')), \
                http.client.INTERNAL_SERVER_ERROR
     if res.status_code != http.client.OK:
+        # hopefully the server has kept a local copy for later recovery
         log.error('msg="Calling WOPI %s failed" url="%s" response="%s"' % (wopicall, wopisrc, res.status_code))
-        # TODO need to save the file on a local storage for later recovery
         return jsonify('Error saving the file, please contact support'), http.client.INTERNAL_SERVER_ERROR
     return None
 

src/core/commoniface.py

@@ -1,13 +1,17 @@
 '''
 commoniface.py
 
-common entities used by all storage interfaces for the IOP WOPI server
+Common entities used by all storage interfaces for the IOP WOPI server.
+Includes functions to store and retrieve Reva-compatible locks.
 
-Author: [email protected], CERN/IT-ST
+Main author: [email protected], CERN/IT-ST
 '''
 
 import time
 import json
+from base64 import urlsafe_b64encode, urlsafe_b64decode
+from binascii import Error as B64Error
+
 
 # standard file missing message
 ENOENT_MSG = 'No such file or directory'
@@ -19,8 +23,56 @@
 ACCESS_ERROR = 'Operation not permitted'
 
 # name of the xattr storing the Reva lock
-LOCKKEY = 'user.iop.lock'
+LOCKKEY = 'iop.lock'
+
+# the prefix used for the lock-id payload to be WebDAV compatible:
+# see https://github.com/cs3org/wopiserver/pull/51#issuecomment-1038798545 for more details;
+# the UUID is fully random and hard coded to identify WOPI locks, no need to have it dynamic
+WEBDAV_LOCK_PREFIX = 'opaquelocktoken:797356a8-0500-4ceb-a8a0-c94c8cde7eba'
+
+# reference to global config
+config = None
+
+
+# Manipulate Reva-compliant locks, i.e. JSON structs with the following format:
+#{
+#   "lock_id": "id1234",
+#   "type": 2,
+#   "user": {
+#      "idp": "https://your-idprovider.org",
+#      "opaque_id": "username",
+#      "type": 1
+#   },
+#   "app_name": "your_app",
+#   "expiration": {
+#      "seconds": 1665446400
+#   }
+#}
 
 def genrevalock(appname, value):
-    '''Return a JSON-formatted lock compatible with the Reva implementation of the CS3 Lock API'''
-    return json.dumps({'h': appname if appname else 'wopi', 't': int(time.time()), 'md': value})
+    '''Return a base64-encoded lock compatible with the Reva implementation of the CS3 Lock API
+       cf. https://github.com/cs3org/cs3apis/blob/main/cs3/storage/provider/v1beta1/resources.proto'''
+    return urlsafe_b64encode(json.dumps(
+        {'lock_id': value,
+         'type': 2,    # LOCK_TYPE_WRITE
+         'app_name': appname if appname else 'wopi',
+         'user': {},
+         'expiration': {
+             'seconds': int(time.time()) + config.getint('general', 'wopilockexpiration')
+         },
+        }).encode()).decode()
+
+
+def retrieverevalock(rawlock):
+    '''Restores the JSON payload from a base64-encoded Reva lock'''
+    try:
+        l = json.loads(urlsafe_b64decode(rawlock).decode())
+        if 'h' in l:
+            # temporary code to support the data structure from WOPI 8.0
+            l['app_name'] = l['h']
+            l['lock_id'] = WEBDAV_LOCK_PREFIX + ' ' + l['md']
+            l['expiration'] = {}
+            l['expiration']['seconds'] = l['exp']
+        return l
+    except (B64Error, json.JSONDecodeError) as e:
+        raise IOError(e)