Ska api fixes

.gitignore

@@ -1 +1,5 @@
 __pycache__
+/build/
+/.eggs
+/record.txt
+/*.egg-info

kadi_apps/app.py

@@ -4,6 +4,7 @@
 """
 
 import logging
+import argparse
 
 from pathlib import Path
 
@@ -46,6 +47,17 @@ def get_app(name=__name__, settings='devel'):
     return app
 
 
-if __name__ == "__main__":
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--unit-test', action='store_const', const='unit_test', dest='settings', default='devel')
+    return parser
+
+
+def main():
     # this starts the development server
-    get_app(settings='devel').run(host='0.0.0.0')
+    args = get_parser().parse_args()
+    get_app(settings=args.settings).run(host='0.0.0.0')
+
+
+if __name__ == "__main__":
+    main()

kadi_apps/blueprints/ska_api/api.py

@@ -6,10 +6,15 @@
 
 
 APPS = {
+    ('agasc',): ['get_star', 'get_stars', 'get_agasc_cone'],
     ('mica', 'starcheck'): ['get_*'],
+    ('mica', 'archive', 'aca_dark', 'dark_cal'): [
+        'get_dark_cal_id', 'get_dark_cal_ids', 'get_dark_cal_image', 'get_dark_cal_props'
+    ],
     ('kadi', 'events'): ['*.filter'],
-    ('kadi', 'commands'): ['get_cmds'],
-    ('kadi', 'commands', 'states'): ['get_states']}
+    ('kadi', 'commands'): ['get_cmds', 'get_observations', 'get_starcats'],
+    ('kadi', 'commands', 'states'): ['get_states']
+}
 
 
 blueprint = Blueprint('ska_api', __name__, template_folder='templates')
@@ -21,23 +26,33 @@ def show_help():
     return render_template('help.html')
 
 
+class NotFound(Exception):
+    pass
+
+
 @blueprint.route("/<path:path>")
 def api(path):
-    logger = logging.getLogger()
+    logger = logging.getLogger('kadi_apps')
     try:
         app_func = _get_function(path)
-        app_kwargs = _get_args(exclude=['table_format'])
+        app_kwargs = _get_args(exclude=[])
+        table_format = app_kwargs.pop('table_format', None)
+        strict_encode = app_kwargs.pop('strict_encode', True)
+
         logger.info(f'{path.replace("/", ".")}(')
         logger.info(f'    **{app_kwargs}')
         logger.info(f')')
 
-        dumper = APIEncoder(request.args.get('table_format', None))
+        dumper = APIEncoder(table_format=table_format, strict_encode=strict_encode)
         output = app_func(**app_kwargs)
         output = dumper.encode(output).encode('utf-8')
         return output, 200
+    except NotFound as e:
+        logger.info(f'NotFound: {e}')
+        return {'ok': False, 'error': str(e)}, 404
     except Exception as e:
         logger.info(f'Exception: {e}')
-        return {'ok': False, 'error': str(e)}
+        return {'ok': False, 'error': str(e)}, 500
 
 
 def _get_function(path):
@@ -65,16 +80,18 @@ def _get_function(path):
             func_parts = module_func_parts[ii + 1:]
 
     if app_module is None:
-        raise ValueError('no app module found for URL path {}'.format(path))
+        raise NotFound('no app module found for URL path {}'.format(path))
 
     if not func_parts:
-        raise ValueError('no function parts found for URL path {}'.format(path))
+        raise NotFound('no function parts found for URL path {}'.format(path))
 
     # Check that the function is allowed
     func_name = '.'.join(func_parts)
     if not any(fnmatch(func_name, app_glob) for app_glob in app_globs):
-        raise ValueError('function {} is not allowed for {}'
-                         .format(func_name, '.'.join(app_module_parts)))
+        raise NotFound(
+            'function {} was not found or is not allowed for {}'
+            .format(func_name, '.'.join(app_module_parts))
+        )
 
     app_func = app_module
     for func_part in func_parts:
@@ -98,9 +115,10 @@ def _get_args(exclude):
 
 
 class APIEncoder(json.JSONEncoder):
-    def __init__(self, table_format=None):
+    def __init__(self, table_format=None, strict_encode=True, **kwargs):
         self.table_format = table_format or 'rows'
         super(APIEncoder, self).__init__()
+        self.strict_encode = strict_encode
 
     def encode_table(self, obj):
         if self.table_format not in ('rows', 'columns'):
@@ -117,26 +135,45 @@ def encode_table(self, obj):
 
     def default(self, obj):
         from astropy.table import Table
+        import numpy as np
 
         # Potentially convert something with a `table` property to an astropy Table.
         if hasattr(obj, 'table') and isinstance(obj.__class__.table, property):
             obj_table = obj.table
             if isinstance(obj_table, Table):
                 obj = obj_table
 
-        if isinstance(obj, Table):
-            out = self.encode_table(obj)
+        if type(obj) in [np.int32, np.int64]:
+            return int(obj)
+
+        elif type(obj) in [np.float32, np.float64]:
+            return float(obj)
+
+        elif isinstance(obj, np.ma.MaskedArray):
+            return {
+                'data': obj.tolist(),
+                'mask': obj.mask.tolist()
+            }
+
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+
+        elif isinstance(obj, Table):
+            return self.encode_table(obj)
 
         elif isinstance(obj, bytes):
-            out = obj.decode('utf-8')
+            return obj.decode('utf-8')
 
         else:
             try:
-                print('OOPS')
                 out = super(APIEncoder, self).default(obj)
             except TypeError:
-                # Last gasp to never fail the JSON encoding.  This is mostly helpful
-                # for debugging instead of the inscrutable exception:
-                # TypeError: default() missing 1 required positional argument: 'o'
-                out = repr(obj)
+                if not self.strict_encode:
+                    # Last gasp to never fail the JSON encoding.  This is mostly helpful
+                    # for debugging instead of an inscrutable exception
+                    out = repr(obj)
+                else:
+                    # re-raise the same exception. This will cause a 500 error (as it should).
+                    # The entrypoint can decide to include the exception message in the response.
+                    raise
         return out

kadi_apps/blueprints/ska_api/templates/help.html

@@ -375,15 +375,15 @@ <h1 class="title">Web-kadi API help</h1>
 <h1>API URL syntax</h1>
 <p>The general URL syntax for querying data via the API interface is as follows:</p>
 <pre class="literal-block">
-http://web-kadi-test.cfa.harvard.edu/api/&lt;package&gt;/&lt;module&gt;/&lt;function&gt;?&lt;arg1&gt;=&lt;val1&gt;&amp;&lt;arg2&gt;=val2...
+  {{ url_for('ska_api.api', path='asdasdasd', _external=True) }}/&lt;package&gt;/&lt;module&gt;/&lt;function&gt;?&lt;arg1&gt;=&lt;val1&gt;&amp;&lt;arg2&gt;=val2...
 </pre>
 <p>This roughly equivalent to the Python pseudo-code:</p>
 <pre class="literal-block">
 from &lt;package&gt;.&lt;module&gt; import &lt;function&gt;
 &lt;function&gt;(arg1=val1, arg2=val2)
 </pre>
 <p>For example:</p>
-<p><a class="reference external" href="http://web-kadi-test.cfa.harvard.edu/api/kadi/events/manvrs/filter?start=2019:001&amp;stop=2019:002">http://web-kadi-test.cfa.harvard.edu/api/kadi/events/manvrs/filter?start=2019:001&amp;stop=2019:002</a></p>
+<p><a class="reference external" href="{{ url_for('ska_api.api', path='kadi/events/manvrs/filter') }}?start=2019:001&amp;stop=2019:002">{{ url_for('ska_api.api', path='kadi/events/manvrs/filter', _external=True) }}?start=2019:001&amp;stop=2019:002</a></p>
 <p>This is equivalent to the Python code:</p>
 <pre class="literal-block">
 from kadi.events import manvrs
@@ -404,14 +404,14 @@ <h2>Table format</h2>
 either a list of dicts (<tt class="docutils literal">rows</tt>) or a dict of lists (<tt class="docutils literal">columns</tt>).  For large query results the
 <tt class="docutils literal">columns</tt> option will generally be more compact because the table column names are not repeated for
 every row. For example:</p>
-<p><a class="reference external" href="http://web-kadi-test.cfa.harvard.edu/api/kadi/events/manvrs/filter?start=2019:001&amp;stop=2019:002&amp;table_format=columns">http://web-kadi-test.cfa.harvard.edu/api/kadi/events/manvrs/filter?start=2019:001&amp;stop=2019:002&amp;table_format=columns</a></p>
+<p><a class="reference external" href="{{ url_for('ska_api.api', path='kadi/events/manvrs/filter') }}?start=2019:001&amp;stop=2019:002&amp;table_format=columns">{{ url_for('ska_api.api', path='kadi/events/manvrs/filter', _external=True) }}?start=2019:001&amp;stop=2019:002&amp;table_format=columns</a></p>
 </div>
 <div class="section" id="timing">
 <h2>Timing</h2>
 <p>A second special option is <tt class="docutils literal">report_timing</tt>, which returns timing information on the query
 instead of the query data.  This may be useful for characterizing the performance of the
 web service (which is not very speedy).  Example:</p>
-<p><a class="reference external" href="http://web-kadi-test.cfa.harvard.edu/api/kadi/events/manvrs/filter?start=2019:001&amp;stop=2019:002&amp;report_timing=1">http://web-kadi-test.cfa.harvard.edu/api/kadi/events/manvrs/filter?start=2019:001&amp;stop=2019:002&amp;report_timing=1</a></p>
+<p><a class="reference external" href="{{ url_for('ska_api.api', path='kadi/events/manvrs/filter') }}?start=2019:001&amp;stop=2019:002&amp;report_timing=1">{{ url_for('ska_api.api', path='kadi/events/manvrs/filter', _external=True) }}?start=2019:001&amp;stop=2019:002&amp;report_timing=1</a></p>
 </div>
 </div>
 <div class="section" id="available-apis">

kadi_apps/tests/conftest.py

@@ -10,16 +10,25 @@ def _run_app():
 
 @pytest.fixture(scope="session")
 def test_server(request):
-    print('kadi_apps.tests.conftest Starting Flask App')
+    import requests
     from multiprocessing import Process
-    p = Process(target=_run_app)
-    p.start()
-    time.sleep(2)  # this is to let the server spin up
     info = {
         'url': 'http://127.0.0.1:5000',
         'user': 'test_user',
         'password': 'test_password',
     }
+    print('kadi_apps.tests.conftest Starting Flask App')
+    p = Process(target=_run_app)
+    p.start()
+    r = None
+    for i in range(20):
+        try:
+            r = requests.get(info['url'])
+            break  # if there is any response we call it a success
+        except Exception:
+            time.sleep(0.2)
+    if r is None:
+        print(f'Test server failed to start: {r}')
     yield info
     print('kadi_apps.tests.conftest Killing Flask App')
     p.kill()