Added non-git source puller functionality

nbgitpuller/__init__.py

@@ -4,6 +4,16 @@
 from notebook.utils import url_path_join
 from tornado.web import StaticFileHandler
 import os
+import nest_asyncio
+
+REPO_PARENT_DIR = None
+TEMP_DOWNLOAD_REPO_DIR = "/tmp/temp_download_repo"
+CACHED_ORIGIN_NON_GIT_REPO = ".nbgitpuller/targets/"
+
+# this allows us to nest usage of the event_loop from asyncio
+# being used by tornado in jupyter distro
+# Ref: https://medium.com/@vyshali.enukonda/how-to-get-around-runtimeerror-this-event-loop-is-already-running-3f26f67e762e
+nest_asyncio.apply()
 
 
 def _jupyter_server_extension_paths():

nbgitpuller/handlers.py

@@ -1,7 +1,6 @@
 from tornado import gen, web, locks
 import traceback
 import urllib.parse
-
 from notebook.base.handlers import IPythonHandler
 import threading
 import json
@@ -11,6 +10,9 @@
 
 from .pull import GitPuller
 from .version import __version__
+from . import hookspecs
+import pluggy
+import nbgitpuller
 
 
 class SyncHandler(IPythonHandler):
@@ -38,6 +40,38 @@ def emit(self, data):
         self.write('data: {}\n\n'.format(serialized_data))
         yield self.flush()
 
+    def setup_plugins(self, provider):
+        pm = pluggy.PluginManager("nbgitpuller")
+        pm.add_hookspecs(hookspecs)
+        pm.load_setuptools_entrypoints("nbgitpuller", name=provider)
+        return pm
+
+    @gen.coroutine
+    def progress_loop(self, queue):
+        while True:
+            try:
+                progress = queue.get_nowait()
+            except Empty:
+                yield gen.sleep(0.1)
+                continue
+            if progress is None:
+                yield gen.sleep(5)
+                return
+            if isinstance(progress, Exception):
+                self.emit({
+                    'phase': 'error',
+                    'message': str(progress),
+                    'output': '\n'.join([
+                        line.strip()
+                        for line in traceback.format_exception(
+                            type(progress), progress, progress.__traceback__
+                        )
+                    ])
+                })
+                return
+
+            self.emit({'output': progress, 'phase': 'syncing'})
+
     @web.authenticated
     @gen.coroutine
     def get(self):
@@ -53,6 +87,7 @@ def get(self):
         try:
             repo = self.get_argument('repo')
             branch = self.get_argument('branch', None)
+            provider = self.get_argument('provider', None)
             depth = self.get_argument('depth', None)
             if depth:
                 depth = int(depth)
@@ -65,16 +100,31 @@ def get(self):
             # so that all repos are always in scope after cloning. Sometimes
             # server_root_dir will include things like `~` and so the path
             # must be expanded.
-            repo_parent_dir = os.path.join(os.path.expanduser(self.settings['server_root_dir']),
-                                           os.getenv('NBGITPULLER_PARENTPATH', ''))
-            repo_dir = os.path.join(repo_parent_dir, self.get_argument('targetpath', repo.split('/')[-1]))
+            repo_parent_dir = os.path.join(os.path.expanduser(self.settings['server_root_dir']), os.getenv('NBGITPULLER_PARENTPATH', ''))
+            nbgitpuller.REPO_PARENT_DIR = repo_parent_dir
+
+            repo_dir = os.path.join(
+                        repo_parent_dir,
+                        self.get_argument('targetpath', repo.split('/')[-1]))
 
             # We gonna send out event streams!
             self.set_header('content-type', 'text/event-stream')
             self.set_header('cache-control', 'no-cache')
 
-            gp = GitPuller(repo, repo_dir, branch=branch, depth=depth, parent=self.settings['nbapp'])
+            # if provider is specified then we are dealing with compressed
+            # archive and not a git repo
+            if provider is not None:
+                pm = self.setup_plugins(provider)
+                req_args = {k: v[0].decode() for k, v in self.request.arguments.items()}
+                download_q = Queue()
+                req_args["progress_func"] = lambda: self.progress_loop(download_q)
+                req_args["download_q"] = download_q
+                hf_args = {"query_line_args": req_args}
+                results = pm.hook.handle_files(**hf_args)
+                repo_dir = repo_parent_dir + results["unzip_dir"]
+                repo = "file://" + results["origin_repo_path"]
 
+            gp = GitPuller(repo, repo_dir, branch=branch, depth=depth, parent=self.settings['nbapp'])
             q = Queue()
 
             def pull():
@@ -87,33 +137,11 @@ def pull():
                     q.put_nowait(e)
                     raise e
             self.gp_thread = threading.Thread(target=pull)
-
             self.gp_thread.start()
-
-            while True:
-                try:
-                    progress = q.get_nowait()
-                except Empty:
-                    yield gen.sleep(0.5)
-                    continue
-                if progress is None:
-                    break
-                if isinstance(progress, Exception):
-                    self.emit({
-                        'phase': 'error',
-                        'message': str(progress),
-                        'output': '\n'.join([
-                            line.strip()
-                            for line in traceback.format_exception(
-                                type(progress), progress, progress.__traceback__
-                            )
-                        ])
-                    })
-                    return
-
-                self.emit({'output': progress, 'phase': 'syncing'})
-
+            self.progress_loop(q)
+            yield gen.sleep(3)
             self.emit({'phase': 'finished'})
+
         except Exception as e:
             self.emit({
                 'phase': 'error',
@@ -147,18 +175,18 @@ def initialize(self):
     @gen.coroutine
     def get(self):
         app_env = os.getenv('NBGITPULLER_APP', default='notebook')
-
         repo = self.get_argument('repo')
         branch = self.get_argument('branch', None)
         depth = self.get_argument('depth', None)
+        provider = self.get_argument('provider', None)
         urlPath = self.get_argument('urlpath', None) or \
-                  self.get_argument('urlPath', None)
+            self.get_argument('urlPath', None)
         subPath = self.get_argument('subpath', None) or \
-                  self.get_argument('subPath', '.')
+            self.get_argument('subPath', '.')
         app = self.get_argument('app', app_env)
         parent_reldir = os.getenv('NBGITPULLER_PARENTPATH', '')
         targetpath = self.get_argument('targetpath', None) or \
-                     self.get_argument('targetPath', repo.split('/')[-1])
+            self.get_argument('targetPath', repo.split('/')[-1])
 
         if urlPath:
             path = urlPath
@@ -171,10 +199,19 @@ def get(self):
             else:
                 path = 'tree/' + path
 
+        if provider is not None:
+            path = "tree/"
+
         self.write(
             self.render_template(
                 'status.html',
-                repo=repo, branch=branch, path=path, depth=depth, targetpath=targetpath, version=__version__
+                repo=repo,
+                branch=branch,
+                path=path,
+                depth=depth,
+                provider=provider,
+                targetpath=targetpath,
+                version=__version__
             ))
         self.flush()
 
@@ -209,3 +246,10 @@ def get(self):
         )
 
         self.redirect(new_url)
+
+
+class ThreadWithResult(threading.Thread):
+    def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, *, daemon=None):
+        def function():
+            self.result = target(*args, **kwargs)
+        super().__init__(group=group, target=function, name=name, daemon=daemon)

nbgitpuller/hookspecs.py

@@ -0,0 +1,22 @@
+import pluggy
+
+hookspec = pluggy.HookspecMarker("nbgitpuller")
+hookimpl = pluggy.HookimplMarker("nbgitpuller")
+
+
+@hookspec(firstresult=True)
+def handle_files(query_line_args):
+    """
+    :param json query_line_args: this includes any argument you put on the url
+    :return two parameter json unzip_dir and origin_repo_path
+    :rtype json object
+
+    The developer uses this function to download, un-compress and save the
+    source files to the TEMP_DOWNLOAD_REPO_DIR folder.
+
+    The parameter, query_line_args, is any argument you put on the URL
+
+    Once the files are saved to the directly, git puller can handle all the
+    standard functions needed to make sure source files are updated or created
+    as needed.
+    """