From 6347264a8fd0b86bf6d0b7b5fc857aaf449003cd Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 16 Dec 2019 17:27:54 +0100 Subject: [PATCH 01/19] added watchdog example --- .travis.yml | 2 +- Dockerfile | 2 +- emu/__init__.py | 4 +- emu/cli.py | 180 -------------------------------- emu/templates/pywps.cfg | 27 ----- emu/wsgi.py | 17 --- environment.yml | 8 +- etc/debug.cfg | 2 - etc/demo.cfg | 2 - etc/sample-custom.cfg | 6 -- etc/sample-postgres.cfg | 3 - emu/default.cfg => pywps.cfg | 6 +- requirements.txt | 3 - setup.py | 6 +- tests/test_wps_inout.py | 4 +- tests/test_wps_ncmeta.py | 8 +- tests/test_wps_poly_centroid.py | 12 +-- tests/test_wps_wordcounter.py | 5 +- 18 files changed, 28 insertions(+), 269 deletions(-) delete mode 100644 emu/cli.py delete mode 100644 emu/templates/pywps.cfg delete mode 100644 emu/wsgi.py delete mode 100644 etc/debug.cfg delete mode 100644 etc/demo.cfg delete mode 100644 etc/sample-custom.cfg delete mode 100644 etc/sample-postgres.cfg rename emu/default.cfg => pywps.cfg (89%) diff --git a/.travis.yml b/.travis.yml index 877a999..09acc69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ install: - python setup.py install before_script: # Start WPS service on port 5000 on 0.0.0.0 - - emu start --daemon --bind-host 0.0.0.0 --port 5000 + - pywps start --bind-host 0.0.0.0 -c pywps.cfg & - sleep 2 script: - pytest -v -m 'not online' tests/ diff --git a/Dockerfile b/Dockerfile index 013e839..e68b7c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN ["/bin/bash", "-c", "source activate wps && python setup.py develop"] # Start WPS service on port 5000 on 0.0.0.0 EXPOSE 5000 ENTRYPOINT ["/bin/bash", "-c"] -CMD ["source activate wps && exec emu start -b 0.0.0.0 -c /opt/wps/etc/demo.cfg"] +CMD ["source activate wps && exec pywps start -b 0.0.0.0 -c /opt/wps/pywps.cfg"] # docker build -t birdhouse/emu . # docker run -p 5000:5000 birdhouse/emu diff --git a/emu/__init__.py b/emu/__init__.py index 6ae11b3..ad4f4ea 100644 --- a/emu/__init__.py +++ b/emu/__init__.py @@ -4,4 +4,6 @@ from .__version__ import __author__, __email__, __version__ -from .wsgi import application +from pywps.application import make_app + +application = make_app() diff --git a/emu/cli.py b/emu/cli.py deleted file mode 100644 index 2004be6..0000000 --- a/emu/cli.py +++ /dev/null @@ -1,180 +0,0 @@ -########################################################### -# Demo WPS service for testing and debugging. -# -# See the werkzeug documentation on how to use the debugger: -# http://werkzeug.pocoo.org/docs/0.12/debug/ -########################################################### - -import os -import psutil -import click -from jinja2 import Environment, PackageLoader -from pywps import configuration - -from . import wsgi -from urllib.parse import urlparse - -PID_FILE = os.path.abspath(os.path.join(os.path.curdir, "pywps.pid")) - -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) - -template_env = Environment( - loader=PackageLoader('emu', 'templates'), - autoescape=True -) - - -def write_user_config(**kwargs): - config_templ = template_env.get_template('pywps.cfg') - rendered_config = config_templ.render(**kwargs) - config_file = os.path.abspath(os.path.join(os.path.curdir, ".custom.cfg")) - with open(config_file, 'w') as fp: - fp.write(rendered_config) - return config_file - - -def get_host(): - url = configuration.get_config_value('server', 'url') - url = url or 'http://localhost:5000/wps' - - click.echo("starting WPS service on {}".format(url)) - - parsed_url = urlparse(url) - if ':' in parsed_url.netloc: - host, port = parsed_url.netloc.split(':') - port = int(port) - else: - host = parsed_url.netloc - port = 80 - return host, port - - -def run_process_action(action=None): - """Run an action with psutil on current process - and return a status message.""" - action = action or 'status' - try: - with open(PID_FILE, 'r') as fp: - pid = int(fp.read()) - p = psutil.Process(pid) - if action == 'stop': - p.terminate() - msg = "pid={}, status=terminated".format(p.pid) - else: - from psutil import _pprint_secs - msg = "pid={}, status={}, created={}".format( - p.pid, p.status(), _pprint_secs(p.create_time())) - if action == 'stop': - os.remove(PID_FILE) - except IOError: - msg = 'No PID file found. Service not running? Try "netstat -nlp | grep :5000".' - except psutil.NoSuchProcess as e: - msg = e.msg - click.echo(msg) - - -def _run(application, bind_host=None, daemon=False): - from werkzeug.serving import run_simple - # call this *after* app is initialized ... needs pywps config. - host, port = get_host() - bind_host = bind_host or host - # need to serve the wps outputs - static_files = { - '/outputs': configuration.get_config_value('server', 'outputpath') - } - run_simple( - hostname=bind_host, - port=port, - application=application, - use_debugger=False, - use_reloader=False, - threaded=True, - # processes=2, - use_evalex=not daemon, - static_files=static_files) - - -@click.group(context_settings=CONTEXT_SETTINGS) -@click.version_option() -def cli(): - """Command line to start/stop a PyWPS service. - - Do not use this service in a production environment. - It's intended to be running in a test environment only! - For more documentation, visit http://pywps.org/doc - """ - pass - - -@cli.command() -def status(): - """Show status of PyWPS service""" - run_process_action(action='status') - - -@cli.command() -def stop(): - """Stop PyWPS service""" - run_process_action(action='stop') - - -@cli.command() -@click.option('--config', '-c', metavar='PATH', help='path to pywps configuration file.') -@click.option('--bind-host', '-b', metavar='IP-ADDRESS', default='127.0.0.1', - help='IP address used to bind service.') -@click.option('--daemon', '-d', is_flag=True, help='run in daemon mode.') -@click.option('--hostname', metavar='HOSTNAME', default='localhost', help='hostname in PyWPS configuration.') -@click.option('--port', metavar='PORT', default='5000', help='port in PyWPS configuration.') -@click.option('--maxsingleinputsize', default='200mb', help='maxsingleinputsize in PyWPS configuration.') -@click.option('--maxprocesses', metavar='INT', default='10', help='maxprocesses in PyWPS configuration.') -@click.option('--parallelprocesses', metavar='INT', default='2', help='parallelprocesses in PyWPS configuration.') -@click.option('--log-level', metavar='LEVEL', default='INFO', help='log level in PyWPS configuration.') -@click.option('--log-file', metavar='PATH', default='pywps.log', help='log file in PyWPS configuration.') -@click.option('--database', default='sqlite:///pywps-logs.sqlite', help='database in PyWPS configuration') -def start(config, bind_host, daemon, hostname, port, - maxsingleinputsize, maxprocesses, parallelprocesses, - log_level, log_file, database): - """Start PyWPS service. - This service is by default available at http://localhost:5000/wps - """ - if os.path.exists(PID_FILE): - click.echo('PID file exists: "{}". Service still running?'.format(PID_FILE)) - os._exit(0) - cfgfiles = [] - cfgfiles.append(write_user_config( - wps_hostname=hostname, - wps_port=port, - wps_maxsingleinputsize=maxsingleinputsize, - wps_maxprocesses=maxprocesses, - wps_parallelprocesses=parallelprocesses, - wps_log_level=log_level, - wps_log_file=log_file, - wps_database=database, - )) - if config: - cfgfiles.append(config) - app = wsgi.create_app(cfgfiles) - # let's start the service ... - # See: - # * https://github.com/geopython/pywps-flask/blob/master/demo.py - # * http://werkzeug.pocoo.org/docs/0.14/serving/ - if daemon: - # daemon (fork) mode - pid = None - try: - pid = os.fork() - if pid: - click.echo('forked process id: {}'.format(pid)) - with open(PID_FILE, 'w') as fp: - fp.write("{}".format(pid)) - except OSError as e: - raise Exception("%s [%d]" % (e.strerror, e.errno)) - - if pid == 0: - os.setsid() - _run(app, bind_host=bind_host, daemon=True) - else: - os._exit(0) - else: - # no daemon - _run(app, bind_host=bind_host) diff --git a/emu/templates/pywps.cfg b/emu/templates/pywps.cfg deleted file mode 100644 index 20951f8..0000000 --- a/emu/templates/pywps.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[server] -{% if wps_url %} -url = {{ wps_url }} -{% else %} -url = http://{{ wps_hostname }}:{{ wps_port }}/wps -{% endif %} -{% if wps_outputurl %} -outputurl = {{ wps_outputurl }} -{% else %} -outputurl = http://{{ wps_hostname }}:{{ wps_port }}/outputs -{% endif %} -allowedinputpaths = / -maxsingleinputsize = {{ wps_maxsingleinputsize|default('200mb') }} -maxprocesses = {{ wps_maxprocesses|default('10') }} -parallelprocesses = {{ wps_parallelprocesses|default('2') }} -{% if wps_outputpath %} -outputpath= {{ wps_outputpath }} -{% endif %} -{% if wps_workdir %} -workdir={{ wps_workdir }} -{% endif %} - -[logging] -level = {{ wps_log_level|default('INFO') }} -file = {{ wps_log_file|default('pywps.log') }} -database = {{ wps_database|default('sqlite:///pywps-logs.sqlite') }} -format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s diff --git a/emu/wsgi.py b/emu/wsgi.py deleted file mode 100644 index b9812c3..0000000 --- a/emu/wsgi.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -from pywps.app.Service import Service - -from .processes import processes - - -def create_app(cfgfiles=None): - config_files = [os.path.join(os.path.dirname(__file__), 'default.cfg')] - if cfgfiles: - config_files.extend(cfgfiles) - if 'PYWPS_CFG' in os.environ: - config_files.append(os.environ['PYWPS_CFG']) - service = Service(processes=processes, cfgfiles=config_files) - return service - - -application = create_app() diff --git a/environment.yml b/environment.yml index 115274b..23d1085 100644 --- a/environment.yml +++ b/environment.yml @@ -4,10 +4,7 @@ channels: - defaults dependencies: - pip -#- pywps=4.2 -- jinja2 -- click -- psutil +# - pywps=4.2 - defusedxml # opendap support - netcdf4 @@ -15,5 +12,6 @@ dependencies: # tests - pytest - pip: - - -e git+https://github.com/geopython/pywps@master#egg=pywps + #- -e git+https://github.com/geopython/pywps@master#egg=pywps + - -e git+https://github.com/cehbrecht/pywps@watchdog#egg=pywps - geomet diff --git a/etc/debug.cfg b/etc/debug.cfg deleted file mode 100644 index c00a2b7..0000000 --- a/etc/debug.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[logging] -level = DEBUG diff --git a/etc/demo.cfg b/etc/demo.cfg deleted file mode 100644 index 52145a3..0000000 --- a/etc/demo.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[logging] -level = WARN diff --git a/etc/sample-custom.cfg b/etc/sample-custom.cfg deleted file mode 100644 index 7b80725..0000000 --- a/etc/sample-custom.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[server] -url = http://demo.org:5000/wps -outputurl = http://demo.org:5000/outputs - -[logging] -level = DEBUG diff --git a/etc/sample-postgres.cfg b/etc/sample-postgres.cfg deleted file mode 100644 index a16b904..0000000 --- a/etc/sample-postgres.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[logging] -level = INFO -database = postgresql+psycopg2://postgres:postgres@localhost:5432/postgres diff --git a/emu/default.cfg b/pywps.cfg similarity index 89% rename from emu/default.cfg rename to pywps.cfg index fb8eafb..6fdf664 100644 --- a/emu/default.cfg +++ b/pywps.cfg @@ -7,16 +7,20 @@ provider_name = Emu provider_url=http://emu.readthedocs.org/en/latest/ [server] +processes = emu.processes.processes url = http://localhost:5000/wps outputurl = http://localhost:5000/outputs allowedinputpaths = / maxsingleinputsize = 200mb maxprocesses = 10 parallelprocesses = 2 +storagetype = file [logging] level = INFO file = pywps.log -#database = sqlite:///:memory: database = sqlite:///pywps-logs.sqlite format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s + +[watchdog] +pause = 2 diff --git a/requirements.txt b/requirements.txt index e807be8..46b0482 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,3 @@ pywps>=4.2.0 -jinja2 -click -psutil defusedxml geomet diff --git a/setup.py b/setup.py index 44b72ec..b16bb86 100644 --- a/setup.py +++ b/setup.py @@ -49,8 +49,4 @@ install_requires=reqs, extras_require={ "dev": dev_reqs, # pip install ".[dev]" - }, - entry_points={ - 'console_scripts': [ - 'emu=emu.cli:cli', - ]},) + },) diff --git a/tests/test_wps_inout.py b/tests/test_wps_inout.py index d3f2577..1a075ed 100644 --- a/tests/test_wps_inout.py +++ b/tests/test_wps_inout.py @@ -3,14 +3,14 @@ from pywps import Service from pywps.tests import assert_response_success -from .common import client_for, get_output, resource_file +from .common import client_for, get_output, resource_file, CFG_FILE from emu.processes.wps_inout import InOut NC_FILE_URL = "file://{}".format(resource_file('test.nc')) def test_wps_inout(): - client = client_for(Service(processes=[InOut()])) + client = client_for(Service(processes=[InOut()], cfgfiles=[CFG_FILE])) datainputs = "string=onetwothree;int=7;float=2.0;boolean=0;text=some string;dataset=@xlink:href={}" resp = client.get( service='WPS', request='Execute', version='1.0.0', identifier='inout', diff --git a/tests/test_wps_ncmeta.py b/tests/test_wps_ncmeta.py index 97bd56f..1219375 100644 --- a/tests/test_wps_ncmeta.py +++ b/tests/test_wps_ncmeta.py @@ -3,7 +3,7 @@ from pywps import Service from pywps.tests import assert_response_success -from .common import client_for, resource_file +from .common import client_for, resource_file, CFG_FILE from emu.processes.wps_ncmeta import NCMeta import owslib.wps @@ -14,7 +14,7 @@ @pytest.mark.online def test_wps_ncmeta_opendap(): - client = client_for(Service(processes=[NCMeta()])) + client = client_for(Service(processes=[NCMeta()], cfgfiles=[CFG_FILE])) datainputs = "dataset_opendap=@xlink:href={0}".format(OPENDAP_URL) resp = client.get( service='wps', request='execute', version='1.0.0', @@ -25,7 +25,7 @@ def test_wps_ncmeta_opendap(): @pytest.mark.online def test_wps_ncmeta_netcdf(): - client = client_for(Service(processes=[NCMeta()])) + client = client_for(Service(processes=[NCMeta()], cfgfiles=[CFG_FILE])) datainputs = "dataset=@xlink:href={0}".format(NC_URL) resp = client.get( service='wps', request='execute', version='1.0.0', @@ -35,7 +35,7 @@ def test_wps_ncmeta_netcdf(): def test_wps_ncmeta_file(): - client = client_for(Service(processes=[NCMeta()])) + client = client_for(Service(processes=[NCMeta()], cfgfiles=[CFG_FILE])) datainputs = "dataset=@xlink:href={0}".format(NC_FILE_URL) resp = client.get( service='wps', request='execute', version='1.0.0', diff --git a/tests/test_wps_poly_centroid.py b/tests/test_wps_poly_centroid.py index b5d26c6..be325de 100644 --- a/tests/test_wps_poly_centroid.py +++ b/tests/test_wps_poly_centroid.py @@ -2,14 +2,12 @@ from pywps import Service from pywps.tests import assert_response_success -from .common import client_for, resource_file, TESTS_HOME, WPS, OWS, get_output +from .common import client_for, resource_file, WPS, OWS, get_output, CFG_FILE from emu.processes.wps_poly_centroid import PolyCentroid -cfgfiles = os.path.join(TESTS_HOME, 'test.cfg') - def test_wps_xml_centroid_get(): - client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=cfgfiles)) + client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=[CFG_FILE])) datainputs = "xml=@xlink:href=file://{0}".format(resource_file('poly.xml'),) resp = client.get( service='WPS', request='Execute', version='1.0.0', @@ -20,7 +18,7 @@ def test_wps_xml_centroid_get(): def test_wps_xml_centroid_post(): - client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=cfgfiles)) + client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=[CFG_FILE])) request_doc = WPS.Execute( OWS.Identifier('poly_centroid'), WPS.DataInputs( @@ -38,7 +36,7 @@ def test_wps_xml_centroid_post(): def test_wps_wkt_centroid_get(): wkt = "POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))" - client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=cfgfiles)) + client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=[CFG_FILE])) datainputs = "wkt={}".format(wkt) resp = client.get( service='WPS', request='Execute', version='1.0.0', @@ -50,7 +48,7 @@ def test_wps_wkt_centroid_get(): def test_wps_wkt_centroid_post(): wkt = "POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))" - client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=cfgfiles)) + client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=[CFG_FILE])) request_doc = WPS.Execute( OWS.Identifier('poly_centroid'), WPS.DataInputs( diff --git a/tests/test_wps_wordcounter.py b/tests/test_wps_wordcounter.py index 5c94da6..e600718 100644 --- a/tests/test_wps_wordcounter.py +++ b/tests/test_wps_wordcounter.py @@ -3,18 +3,19 @@ from pywps import Service from pywps.tests import assert_response_success -from .common import client_for, resource_file, get_output +from .common import client_for, resource_file, get_output, CFG_FILE from emu.processes.wps_wordcounter import WordCounter def test_wps_wordcount_file(): - client = client_for(Service(processes=[WordCounter()])) + client = client_for(Service(processes=[WordCounter()], cfgfiles=[CFG_FILE])) datainputs = "text=@xlink:href=file://{0}".format( resource_file('alice-in-wonderland.txt')) resp = client.get( service='wps', request='execute', version='1.0.0', identifier='wordcounter', datainputs=datainputs) + # print(resp.get_data()) assert_response_success(resp) From 2a6bac16433f22bad96241e9d1847b7572a8e776 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Wed, 18 Dec 2019 17:45:48 +0100 Subject: [PATCH 02/19] update config --- pywps.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pywps.cfg b/pywps.cfg index 6fdf664..032fb89 100644 --- a/pywps.cfg +++ b/pywps.cfg @@ -22,5 +22,5 @@ file = pywps.log database = sqlite:///pywps-logs.sqlite format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s -[watchdog] -pause = 2 +[jobqueue] +pause = 30 From acc1da3ce1b872bedef05b07f2770ebae45e1f84 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Thu, 19 Dec 2019 17:53:31 +0100 Subject: [PATCH 03/19] update config --- pywps.cfg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pywps.cfg b/pywps.cfg index 032fb89..0ad6120 100644 --- a/pywps.cfg +++ b/pywps.cfg @@ -13,14 +13,15 @@ outputurl = http://localhost:5000/outputs allowedinputpaths = / maxsingleinputsize = 200mb maxprocesses = 10 +maxprocessingtime = 30 parallelprocesses = 2 storagetype = file [logging] -level = INFO +level = DEBUG file = pywps.log database = sqlite:///pywps-logs.sqlite format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s [jobqueue] -pause = 30 +pause = 2 From 459b9159aec8f6c3f4151238cf3d0d79db0d669f Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Thu, 19 Dec 2019 18:49:31 +0100 Subject: [PATCH 04/19] update config --- pywps.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pywps.cfg b/pywps.cfg index 0ad6120..77ad5b1 100644 --- a/pywps.cfg +++ b/pywps.cfg @@ -18,10 +18,10 @@ parallelprocesses = 2 storagetype = file [logging] -level = DEBUG +level = INFO file = pywps.log database = sqlite:///pywps-logs.sqlite format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s [jobqueue] -pause = 2 +pause = 30 From 13a3e0f93ed965cf3ae6ee650e97179e67257dd4 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Thu, 2 Jan 2020 18:50:21 +0100 Subject: [PATCH 05/19] update dockerfile pywps command --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e68b7c1..c269dac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN ["/bin/bash", "-c", "source activate wps && python setup.py develop"] # Start WPS service on port 5000 on 0.0.0.0 EXPOSE 5000 ENTRYPOINT ["/bin/bash", "-c"] -CMD ["source activate wps && exec pywps start -b 0.0.0.0 -c /opt/wps/pywps.cfg"] +CMD ["source activate wps && exec pywps -c /opt/wps/pywps.cfg start -b 0.0.0.0"] # docker build -t birdhouse/emu . # docker run -p 5000:5000 birdhouse/emu From e352c16c5868051423a27333a41eb7ab67b3ab32 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Thu, 2 Jan 2020 19:22:38 +0100 Subject: [PATCH 06/19] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 09acc69..4dc153e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ install: - python setup.py install before_script: # Start WPS service on port 5000 on 0.0.0.0 - - pywps start --bind-host 0.0.0.0 -c pywps.cfg & + - pywps -c pywps.cfg start --bind-host 0.0.0.0 & - sleep 2 script: - pytest -v -m 'not online' tests/ From b3a3c859f13690839625d83201a2a96311e02373 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Thu, 2 Jan 2020 19:26:31 +0100 Subject: [PATCH 07/19] fix travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4dc153e..23c2a8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,8 @@ install: - conda install pytest flake8 # Install WPS - python setup.py install + # Init DB + - pywps -c pywps.cfg migrate before_script: # Start WPS service on port 5000 on 0.0.0.0 - pywps -c pywps.cfg start --bind-host 0.0.0.0 & From 4590837ddf3076185196e285fc176f8c60a459b1 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Fri, 3 Jan 2020 12:03:53 +0100 Subject: [PATCH 08/19] update makefile --- Makefile | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 0edf4aa..78552c4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Configuration APP_ROOT := $(abspath $(lastword $(MAKEFILE_LIST))/..) -APP_NAME := emu +CFG_FILE ?= pywps.cfg # Bumpversion 'dry' config # if 'dry' is specified as target, any bumpversion call using 'BUMP_XARGS' will not apply changes @@ -25,10 +25,8 @@ help: @echo " help to print this help message. (Default)" @echo " install to install app by running 'pip install -e .'" @echo " develop to install with additional development requirements." - @echo " start to start $(APP_NAME) service as daemon (background process)." - @echo " stop to stop $(APP_NAME) service." - @echo " restart to restart $(APP_NAME) service." - @echo " status to show status of $(APP_NAME) service." + @echo " migrate to upgrade or initialize database." + @echo " start to start pywps service." @echo " clean to remove all files generated by build and tests." @echo "\nTesting targets:" @echo " test to run tests (but skip long running tests)." @@ -62,24 +60,15 @@ develop: @echo "Installing development requirements for tests and docs ..." @-bash -c 'pip install -e ".[dev]"' +.PHONY: migrate +migrate: + @echo "Upgrade or initialize database ..." + @-bash -c 'pywps -c "$(CFG_FILE)" migrate' + .PHONY: start start: @echo "Starting application ..." - @-bash -c "$(APP_NAME) start -d" - -.PHONY: stop -stop: - @echo "Stopping application ..." - @-bash -c "$(APP_NAME) stop" - -.PHONY: restart -restart: stop start - @echo "Restarting application ..." - -.PHONY: status -status: - @echo "Show status ..." - @-bash -c "$(APP_NAME) status" + @-bash -c 'pywps -c "$(CFG_FILE)" start' .PHONY: clean clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts From 2b15468df5bb650cdd4ee996e86cb9b401ea55b5 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Fri, 3 Jan 2020 14:04:41 +0100 Subject: [PATCH 09/19] added db init to dockerfile --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6a58160..5f57250 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,9 @@ RUN conda env create -n wps -f environment.yml # Install WPS RUN ["/bin/bash", "-c", "source activate wps && python setup.py develop"] +# Init DB +RUN ["/bin/bash", "-c", "source activate wps && pywps -c /opt/wps/pywps.cfg migrate"] + # Start WPS service on port 5000 on 0.0.0.0 EXPOSE 5000 ENTRYPOINT ["/bin/bash", "-c"] From 2046efb32c04b457d9cfa3a3882008d519a6b13d Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Fri, 3 Jan 2020 14:23:49 +0100 Subject: [PATCH 10/19] update docs --- docs/source/configuration.rst | 24 +++++-------------- docs/source/installation.rst | 31 +++++++------------------ docs/source/tutorial/using_postgres.rst | 8 +++---- 3 files changed, 19 insertions(+), 44 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e182916..b726582 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -3,34 +3,22 @@ Configuration ============= -Command-line options --------------------- - -You can overwrite the default `PyWPS`_ configuration by using command-line options. -See the Emu help which options are available:: - - $ emu start --help - --hostname HOSTNAME hostname in PyWPS configuration. - --port PORT port in PyWPS configuration. - -Start service with different hostname and port:: - - $ emu start --hostname localhost --port 5001 Use a custom configuration file ------------------------------- You can overwrite the default `PyWPS`_ configuration by providing your own -PyWPS configuration file (just modifiy the options you want to change). -Use one of the existing ``sample-*.cfg`` files as example and copy them to ``etc/custom.cfg``. +PyWPS configuration file. +Use the existing ``pywps.cfg`` file as example and copy it to ``custom.cfg``. For example change the hostname (*demo.org*) and logging level: .. code-block:: sh $ cd emu - $ vim etc/custom.cfg - $ cat etc/custom.cfg + $ cp pywps.cfg custom.cfg + $ vim custom.cfg + $ cat custom.cfg [server] url = http://demo.org:5000/wps outputurl = http://demo.org:5000/outputs @@ -43,7 +31,7 @@ Start the service with your custom configuration: .. code-block:: sh # start the service with this configuration - $ emu start -c etc/custom.cfg + $ pywps -c custom.cfg start .. _PyWPS: http://pywps.org/ diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 41557c8..c749180 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -63,51 +63,38 @@ For development you can use this command: OR $ make develop -Start Emu PyWPS service ------------------------ +Start PyWPS service +------------------- -After successful installation you can start the service using the ``emu`` command-line. +After successful installation you can start the service using the ``pywps`` command-line. .. code-block:: console - $ emu start --help # show help - $ emu start # start service with default configuration - - OR + # show help + $ pywps -h - $ emu start --daemon # start service as daemon - loading configuration - forked process id: 42 + # start service with default configuration (pywps.cfg) + $ pywps -c pywps.cfg start The deployed WPS service is by default available on: http://localhost:5000/wps?service=WPS&version=1.0.0&request=GetCapabilities. -.. NOTE:: Remember the process ID (PID) so you can stop the service with ``kill PID``. - -You can find which process uses a given port using the following command (here for port 5000): - -.. code-block:: console - - $ netstat -nlp | grep :5000 - Check the log files for errors: .. code-block:: console - $ tail -f pywps.log + $ tail -f pywps.log ... or do it the lazy way +++++++++++++++++++++++++ -You can also use the ``Makefile`` to start and stop the service: +You can also use the ``Makefile`` to start and the service: .. code-block:: console $ make start - $ make status $ tail -f pywps.log - $ make stop Run Emu as Docker container --------------------------- diff --git a/docs/source/tutorial/using_postgres.rst b/docs/source/tutorial/using_postgres.rst index 463a95f..06c3bc9 100644 --- a/docs/source/tutorial/using_postgres.rst +++ b/docs/source/tutorial/using_postgres.rst @@ -40,20 +40,20 @@ for this database is:: # postgresql+psycopg2://user:password@host:port/dbname postgresql+psycopg2://postgres:postgres@localhost:5432/postgres -Configure this connection string in ``etc/custom.cfg``, ``logging`` section, ``database`` option: +Configure this connection string in ``custom.cfg``, ``logging`` section, ``database`` option: .. code-block:: console - $ vim etc/custom.cfg + $ vim custom.cfg [logging] level = INFO database = postgresql+psycopg2://postgres:postgres@localhost:5432/postgres -Start the emu service: +Start the PyWPS service: .. code-block:: console - $ emu start -c etc/custom.cfg + $ pywps -c custom.cfg start Your Emu WPS service should be available at the following URL: From 5797acb1fffd4cf361e477fabd80923e5dc734a7 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Tue, 7 Jan 2020 17:32:56 +0100 Subject: [PATCH 11/19] added db migrate step to docs --- docs/source/installation.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index c749180..3e4c650 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -29,7 +29,6 @@ Install the ``emu`` Conda package: $ conda install -c birdhouse -c conda-forge emu $ emu --help - Install from GitHub ------------------- @@ -63,6 +62,15 @@ For development you can use this command: OR $ make develop +Initialize Database +------------------- + +Before you can start the service you need to initialize or upgrade the database: + +.. code-block:: console + + $ make migrate + Start PyWPS service ------------------- From 2257d769999818f85b7f21bda3981ca2a93abdd4 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Tue, 7 Jan 2020 18:11:46 +0100 Subject: [PATCH 12/19] added python 3.8 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 23c2a8a..e9c1142 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: python python: - "3.6" - "3.7" + - "3.8" os: - linux # - osx From 0c5e561e912f780ec37f7ed94852e859e6302313 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Fri, 10 Jan 2020 16:09:05 +0100 Subject: [PATCH 13/19] update pywps.cfg --- pywps.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pywps.cfg b/pywps.cfg index 77ad5b1..3be20de 100644 --- a/pywps.cfg +++ b/pywps.cfg @@ -13,7 +13,6 @@ outputurl = http://localhost:5000/outputs allowedinputpaths = / maxsingleinputsize = 200mb maxprocesses = 10 -maxprocessingtime = 30 parallelprocesses = 2 storagetype = file @@ -24,4 +23,4 @@ database = sqlite:///pywps-logs.sqlite format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s [jobqueue] -pause = 30 +pause = 5 From 0b36f1baabaa33c5e16a60438e333d2a44c410b1 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Fri, 10 Jan 2020 16:09:42 +0100 Subject: [PATCH 14/19] pin python 3.6 --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 1f8b677..6ea3501 100644 --- a/environment.yml +++ b/environment.yml @@ -3,6 +3,7 @@ channels: - conda-forge - defaults dependencies: +- python=3.6 - pip # - pywps=4.2 - jinja2 From ef4f8c8812c2a15772c47955f03201aff0b3fee4 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Thu, 23 Jan 2020 18:51:33 +0100 Subject: [PATCH 15/19] pin python 3.7 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6ea3501..2ec1fa8 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: -- python=3.6 +- python=3.7 - pip # - pywps=4.2 - jinja2 From bfbc0c007190828718deb5a4288a9e2ed0241fde Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Thu, 14 May 2020 13:20:08 +0200 Subject: [PATCH 16/19] fixed conda env --- environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 2ec1fa8..e98bf4c 100644 --- a/environment.yml +++ b/environment.yml @@ -16,6 +16,5 @@ dependencies: # tests - pytest - pip: - #- -e git+https://github.com/geopython/pywps@master#egg=pywps - - -e git+https://github.com/cehbrecht/pywps@watchdog#egg=pywps + - -e git+https://github.com/geopython/pywps@master#egg=pywps - geomet From 969d4f08b4204791947c47cba668014fe675d1cf Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Fri, 15 May 2020 13:42:07 +0200 Subject: [PATCH 17/19] fix tests --- setup.cfg | 2 +- tests/test_wps_caps.py | 4 ++-- tests/test_wps_ncmeta.py | 1 + tests/test_wps_ncml.py | 1 + tests/test_wps_translation.py | 8 ++++---- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 31463df..916b9e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ markers = slow: mark test to be slow flaky: mark test to be flaky network: mark test to need network + netcdf: mark test to netcdf [flake8] ignore = F401,E402 @@ -43,4 +44,3 @@ exclude = [doc8] ignore-path = docs/build,docs/source/_templates,docs/source/_static max-line-length = 120 - diff --git a/tests/test_wps_caps.py b/tests/test_wps_caps.py index 376bc93..d308b65 100644 --- a/tests/test_wps_caps.py +++ b/tests/test_wps_caps.py @@ -1,12 +1,12 @@ # -*- encoding: utf-8 -*- from pywps import Service -from .common import client_for +from .common import client_for, CFG_FILE from emu.processes import processes def test_wps_caps(): - client = client_for(Service(processes=processes)) + client = client_for(Service(processes=processes, cfgfiles=CFG_FILE)) resp = client.get(service='wps', request='getcapabilities', version='1.0.0', language='en-US') names = resp.xpath_text('/wps:Capabilities' '/wps:ProcessOfferings' diff --git a/tests/test_wps_ncmeta.py b/tests/test_wps_ncmeta.py index 1219375..fde77ce 100644 --- a/tests/test_wps_ncmeta.py +++ b/tests/test_wps_ncmeta.py @@ -24,6 +24,7 @@ def test_wps_ncmeta_opendap(): @pytest.mark.online +@pytest.mark.netcdf def test_wps_ncmeta_netcdf(): client = client_for(Service(processes=[NCMeta()], cfgfiles=[CFG_FILE])) datainputs = "dataset=@xlink:href={0}".format(NC_URL) diff --git a/tests/test_wps_ncml.py b/tests/test_wps_ncml.py index 9319302..1900d0b 100644 --- a/tests/test_wps_ncml.py +++ b/tests/test_wps_ncml.py @@ -8,6 +8,7 @@ @pytest.mark.online +@pytest.mark.netcdf def test_wps_ncml(): client = client_for(Service(processes=[NcMLAgg()], cfgfiles=CFG_FILE)) diff --git a/tests/test_wps_translation.py b/tests/test_wps_translation.py index af20436..8370336 100644 --- a/tests/test_wps_translation.py +++ b/tests/test_wps_translation.py @@ -1,12 +1,12 @@ from pywps import Service from pywps.tests import assert_response_success -from .common import client_for +from .common import client_for, CFG_FILE from emu.processes.wps_translation import Translation def test_wps_translation_describe_fr(): - client = client_for(Service(processes=[Translation()])) + client = client_for(Service(processes=[Translation()], cfgfiles=CFG_FILE)) resp = client.get( service='wps', request='DescribeProcess', version='1.0.0', identifier='translation', @@ -21,7 +21,7 @@ def test_wps_translation_describe_fr(): def test_wps_translation_describe_de(): - client = client_for(Service(processes=[Translation()])) + client = client_for(Service(processes=[Translation()], cfgfiles=CFG_FILE)) resp = client.get( service='wps', request='DescribeProcess', version='1.0.0', identifier='translation', @@ -36,7 +36,7 @@ def test_wps_translation_describe_de(): def test_wps_translation_execute(): - client = client_for(Service(processes=[Translation()])) + client = client_for(Service(processes=[Translation()], cfgfiles=CFG_FILE)) datainputs = "input1=10" resp = client.get( service='wps', request='execute', version='1.0.0', From 49d6e408be82e3f0b8c07bc1e7bed65ba682923f Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Fri, 15 May 2020 13:43:57 +0200 Subject: [PATCH 18/19] fixed tests --- tests/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/common.py b/tests/common.py index 8c0d48c..d93c01c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -10,6 +10,8 @@ WPS, OWS = get_ElementMakerForVersion(VERSION) xpath_ns = get_xpath_ns(VERSION) +os.environ['PYWPS_CFG'] = CFG_FILE + def resource_file(filepath): return os.path.join(TESTS_HOME, 'testdata', filepath) From f6223e26ba3e6224a18998b207c6935c154608f1 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Fri, 15 May 2020 15:07:27 +0200 Subject: [PATCH 19/19] update test.cfg --- tests/test.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test.cfg b/tests/test.cfg index f092411..3572c61 100644 --- a/tests/test.cfg +++ b/tests/test.cfg @@ -1,3 +1,5 @@ [server] -allowedinputpaths=/ +allowedinputpaths = / +maxprocesses = 10 +parallelprocesses = 2 language = en-US,fr-CA,de-DE