From 3f2209b805f2974098af3c0fcd1f8b7a5eef7143 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Mon, 3 Jun 2024 12:45:28 +0200 Subject: [PATCH 1/8] scripts: compile languages before starting flask --- scripts/flask.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/flask.sh b/scripts/flask.sh index a1b29918..aa4b18be 100755 --- a/scripts/flask.sh +++ b/scripts/flask.sh @@ -1,3 +1,4 @@ #!/bin/bash # Run Flask in debug mode -poetry run flask --app sketch_map_tool/routes.py --debug run --port 8081 +pybabel compile -d sketch_map_tool/translations +flask --app sketch_map_tool/routes.py --debug run --port 8081 From 8c1c144cfb5676916f250b3f3dde3d35a6fecfc9 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 4 Jun 2024 11:49:27 +0200 Subject: [PATCH 2/8] add tick box for consent --- client-src/digitize/index.js | 2 ++ sketch_map_tool/templates/digitize.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client-src/digitize/index.js b/client-src/digitize/index.js index fc775dfd..0c18eae9 100644 --- a/client-src/digitize/index.js +++ b/client-src/digitize/index.js @@ -11,6 +11,8 @@ filebokz(); // disable submit... setDisabled("submitBtn", true); +const consentCheckbox = document.getElementById("consent"); + // ...until files are added const fileElement = document.querySelector(".filebokz"); fileElement.addEventListener("file-added", () => { diff --git a/sketch_map_tool/templates/digitize.html b/sketch_map_tool/templates/digitize.html index 81de1bde..05987dde 100644 --- a/sketch_map_tool/templates/digitize.html +++ b/sketch_map_tool/templates/digitize.html @@ -47,7 +47,9 @@ class="size">)    {{ _('Max total size: 500 MB') }}
+

+ {{ _('By uploading the file(s), you agree that the content of the files and the time of upload will be saved and used for the further development of the service.') }}

From 1c163907a992d1473d32dd0a92042a9167478fbd Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Mon, 3 Jun 2024 12:45:28 +0200 Subject: [PATCH 3/8] scripts: compile languages before starting flask --- scripts/flask.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/flask.sh b/scripts/flask.sh index a1b29918..aa4b18be 100755 --- a/scripts/flask.sh +++ b/scripts/flask.sh @@ -1,3 +1,4 @@ #!/bin/bash # Run Flask in debug mode -poetry run flask --app sketch_map_tool/routes.py --debug run --port 8081 +pybabel compile -d sketch_map_tool/translations +flask --app sketch_map_tool/routes.py --debug run --port 8081 From 31256d74d55f11331c7bf662bf10b99a44e99b20 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 4 Jun 2024 11:49:27 +0200 Subject: [PATCH 4/8] add tick box for consent --- client-src/digitize/index.js | 2 ++ sketch_map_tool/templates/digitize.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client-src/digitize/index.js b/client-src/digitize/index.js index fc775dfd..0c18eae9 100644 --- a/client-src/digitize/index.js +++ b/client-src/digitize/index.js @@ -11,6 +11,8 @@ filebokz(); // disable submit... setDisabled("submitBtn", true); +const consentCheckbox = document.getElementById("consent"); + // ...until files are added const fileElement = document.querySelector(".filebokz"); fileElement.addEventListener("file-added", () => { diff --git a/sketch_map_tool/templates/digitize.html b/sketch_map_tool/templates/digitize.html index 81de1bde..05987dde 100644 --- a/sketch_map_tool/templates/digitize.html +++ b/sketch_map_tool/templates/digitize.html @@ -47,7 +47,9 @@ class="size">)    {{ _('Max total size: 500 MB') }}
+

+ {{ _('By uploading the file(s), you agree that the content of the files and the time of upload will be saved and used for the further development of the service.') }}

From 097ea819fd043122384608747ec7afc70e3f87e2 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Tue, 4 Jun 2024 15:01:43 +0200 Subject: [PATCH 5/8] feat: implement opt-out of sketch map usage --- sketch_map_tool/database/client_flask.py | 20 +++- sketch_map_tool/routes.py | 3 +- sketch_map_tool/templates/digitize.html | 1 - .../cassette/test_generate_quality_report | 104 ++++++++++++++++++ .../integration/test_database_client_flask.py | 6 +- 5 files changed, 124 insertions(+), 10 deletions(-) diff --git a/sketch_map_tool/database/client_flask.py b/sketch_map_tool/database/client_flask.py index d5b79256..d2fbb440 100644 --- a/sketch_map_tool/database/client_flask.py +++ b/sketch_map_tool/database/client_flask.py @@ -85,23 +85,33 @@ def set_async_result_ids(request_uuid, map_: dict[REQUEST_TYPES, str]): _insert_id_map(request_uuid, map_) -def insert_files(files) -> list[int]: +def insert_files(files, consent: bool) -> list[int]: """Insert uploaded files as blob into the database and return primary keys""" create_query = """ CREATE TABLE IF NOT EXISTS blob( id SERIAL PRIMARY KEY, file_name VARCHAR, file BYTEA, - ts timestamp with time zone DEFAULT now() + consent BOOLEAN, + ts TIMESTAMP WITH TIME ZONE DEFAULT now() ) - """ - insert_query = "INSERT INTO blob(file_name, file) VALUES (%s, %s) RETURNING id" + """ + insert_query = ( + "INSERT INTO blob(file_name, file, consent) VALUES (%s, %s, %s) RETURNING id" + ) db_conn = open_connection() with db_conn.cursor() as curs: curs.execute(create_query) ids = [] for file in files: - curs.execute(insert_query, (secure_filename(file.filename), file.read())) + curs.execute( + insert_query, + ( + secure_filename(file.filename), + file.read(), + consent, + ), + ) ids.append(curs.fetchone()[0]) return ids diff --git a/sketch_map_tool/routes.py b/sketch_map_tool/routes.py index 9bb5ef4e..b9c3f63c 100644 --- a/sketch_map_tool/routes.py +++ b/sketch_map_tool/routes.py @@ -121,12 +121,13 @@ def digitize(lang="en") -> str: @app.post("//digitize/results") def digitize_results_post(lang="en") -> Response: """Upload files to create geodata results""" + consent: bool = True # No files uploaded if "file" not in request.files: return redirect(url_for("digitize", lang=lang)) files = request.files.getlist("file") validate_uploaded_sketchmaps(files) - ids = db_client_flask.insert_files(files) + ids = db_client_flask.insert_files(files, consent) file_names = [db_client_flask.select_file_name(i) for i in ids] args = [ upload_processing.read_qr_code(to_array(db_client_flask.select_file(_id))) diff --git a/sketch_map_tool/templates/digitize.html b/sketch_map_tool/templates/digitize.html index 05987dde..74a51bb3 100644 --- a/sketch_map_tool/templates/digitize.html +++ b/sketch_map_tool/templates/digitize.html @@ -47,7 +47,6 @@ class="size">)    {{ _('Max total size: 500 MB') }}
-

{{ _('By uploading the file(s), you agree that the content of the files and the time of upload will diff --git a/tests/fixtures/cassette/test_generate_quality_report b/tests/fixtures/cassette/test_generate_quality_report index f7ffecc1..81f3c978 100644 --- a/tests/fixtures/cassette/test_generate_quality_report +++ b/tests/fixtures/cassette/test_generate_quality_report @@ -4204,4 +4204,108 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Connection: + - keep-alive + User-Agent: + - python-requests/2.32.3 + method: GET + uri: https://maps.heigit.org/sketch-map-tool/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fjpeg&TRANSPARENT=FALSE&LAYERS=world_imagery&WIDTH=1716&HEIGHT=1436&SRS=EPSG%3A3857&STYLES=&BBOX=964445.3646475708%2C6343463.48326091%2C967408.255014792%2C6345943.466874749 + response: + body: + string: ' + + + + 403 Forbidden + + + +

Forbidden

+ +

You don''t have permission to access this resource.

+ +
+ +
Apache/2.4.52 (Ubuntu) Server at maps.heigit.org Port 443
+ + + + ' + headers: + Connection: + - Keep-Alive + Content-Length: + - '281' + Content-Type: + - text/html; charset=iso-8859-1 + Date: + - Mon, 03 Jun 2024 13:06:07 GMT + Keep-Alive: + - timeout=5, max=100 + Server: + - Apache/2.4.52 (Ubuntu) + Strict-Transport-Security: + - max-age=63072000; includeSubdomains; + status: + code: 403 + message: Forbidden +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Connection: + - keep-alive + User-Agent: + - python-requests/2.32.3 + method: GET + uri: https://maps.heigit.org/sketch-map-tool/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=FALSE&LAYERS=world_imagery_fallback&WIDTH=1716&HEIGHT=1436&SRS=EPSG%3A3857&STYLES=&BBOX=964445.3646475708%2C6343463.48326091%2C967408.255014792%2C6345943.466874749 + response: + body: + string: ' + + + + 403 Forbidden + + + +

Forbidden

+ +

You don''t have permission to access this resource.

+ +
+ +
Apache/2.4.52 (Ubuntu) Server at maps.heigit.org Port 443
+ + + + ' + headers: + Connection: + - Keep-Alive + Content-Length: + - '281' + Content-Type: + - text/html; charset=iso-8859-1 + Date: + - Mon, 03 Jun 2024 13:06:08 GMT + Keep-Alive: + - timeout=5, max=100 + Server: + - Apache/2.4.52 (Ubuntu) + Strict-Transport-Security: + - max-age=63072000; includeSubdomains; + status: + code: 403 + message: Forbidden version: 1 diff --git a/tests/integration/test_database_client_flask.py b/tests/integration/test_database_client_flask.py index c15e93fb..af3765a2 100644 --- a/tests/integration/test_database_client_flask.py +++ b/tests/integration/test_database_client_flask.py @@ -29,7 +29,7 @@ def file_ids(files, flask_app): """IDs of uploaded files stored in the database.""" with flask_app.app_context(): # setup - ids = db_client_flask.insert_files(files) + ids = db_client_flask.insert_files(files, consent=True) yield ids # teardown for i in ids: @@ -80,7 +80,7 @@ def test_get_async_result_id(flask_app, uuid): def test_insert_files(flask_app, files): with flask_app.app_context(): - ids = client_flask.insert_files(files) + ids = client_flask.insert_files(files, consent=True) try: assert len(ids) == 2 assert isinstance(ids[0], int) @@ -92,7 +92,7 @@ def test_insert_files(flask_app, files): def test_delete_file(flask_app, files): with flask_app.app_context(): - ids = client_flask.insert_files(files) + ids = client_flask.insert_files(files, consent=True) for i in ids: # No error should be raised client_flask.delete_file(i) From 03e8d330de415768488ef1d8180d5e21e443bc10 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Tue, 4 Jun 2024 17:14:49 +0200 Subject: [PATCH 6/8] feat(digitize): send consent parameter --- .../map_generation/generate_pdf.py | 2 +- sketch_map_tool/routes.py | 6 ++- sketch_map_tool/templates/digitize.html | 2 +- .../cassette/test_generate_sketch_map | 46 ++++++++++++++++++- tests/integration/test_routes.py | 17 +++++++ tests/unit/test_routes_digitize.py | 2 +- 6 files changed, 69 insertions(+), 6 deletions(-) diff --git a/sketch_map_tool/map_generation/generate_pdf.py b/sketch_map_tool/map_generation/generate_pdf.py index a6ef3323..3cd74a99 100644 --- a/sketch_map_tool/map_generation/generate_pdf.py +++ b/sketch_map_tool/map_generation/generate_pdf.py @@ -26,7 +26,7 @@ Image.MAX_IMAGE_PIXELS = None -def generate_pdf( # noqa: C901 +def generate_pdf( map_image_input: PILImage, qr_code: Drawing, format_: PaperFormat, diff --git a/sketch_map_tool/routes.py b/sketch_map_tool/routes.py index b9c3f63c..d4b29352 100644 --- a/sketch_map_tool/routes.py +++ b/sketch_map_tool/routes.py @@ -121,7 +121,11 @@ def digitize(lang="en") -> str: @app.post("//digitize/results") def digitize_results_post(lang="en") -> Response: """Upload files to create geodata results""" - consent: bool = True + # "consent" is a checkbox and value is only send if it is checked + if "consent" in request.form.keys(): + consent: bool = True + else: + consent: bool = False # No files uploaded if "file" not in request.files: return redirect(url_for("digitize", lang=lang)) diff --git a/sketch_map_tool/templates/digitize.html b/sketch_map_tool/templates/digitize.html index 74a51bb3..7cdf2dbb 100644 --- a/sketch_map_tool/templates/digitize.html +++ b/sketch_map_tool/templates/digitize.html @@ -48,7 +48,7 @@

- + {{ _('By uploading the file(s), you agree that the content of the files and the time of upload will be saved and used for the further development of the service.') }}

diff --git a/tests/fixtures/cassette/test_generate_sketch_map b/tests/fixtures/cassette/test_generate_sketch_map index 3d8b4fce..9a57f06d 100644 --- a/tests/fixtures/cassette/test_generate_sketch_map +++ b/tests/fixtures/cassette/test_generate_sketch_map @@ -19947,8 +19947,6 @@ interactions: - '"1871091362"' Strict-Transport-Security: - max-age=63072000 - Transfer-Encoding: - - chunked Vary: - Accept-Encoding Via: @@ -19970,4 +19968,48 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Connection: + - keep-alive + User-Agent: + - python-requests/2.32.3 + method: GET + uri: https://maps.heigit.org/osm-carto/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=FALSE&LAYERS=heigit%3Aosm-carto%402xx&WIDTH=1716&HEIGHT=1436&SRS=EPSG%3A3857&STYLES=&BBOX=964445.3646475708%2C6343463.48326091%2C967408.255014792%2C6345943.466874749 + response: + body: + string: !!binary | + iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhV + AAAAAElFTkSuQmCC + headers: + Access-control-allow-origin: + - '*' + Cache-Control: + - max-age=86400 + Connection: + - Upgrade, Keep-Alive + Content-length: + - '759648' + Content-type: + - image/png + Date: + - Tue, 04 Jun 2024 15:02:02 GMT + Expires: + - Wed, 05 Jun 2024 15:02:02 GMT + Keep-Alive: + - timeout=5, max=100 + Server: + - Apache/2.4.52 (Ubuntu) + Strict-Transport-Security: + - max-age=63072000; includeSubdomains; + Upgrade: + - h2 + status: + code: 200 + message: OK version: 1 diff --git a/tests/integration/test_routes.py b/tests/integration/test_routes.py index 7e82889b..2328fc5f 100644 --- a/tests/integration/test_routes.py +++ b/tests/integration/test_routes.py @@ -17,6 +17,21 @@ def test_create_results_post(params, flask_client): def test_digitize_results_post(sketch_map_marked, flask_client): + data = {"file": [(BytesIO(sketch_map_marked), "sketch_map.png")], "consent": "True"} + response = flask_client.post("/digitize/results", data=data, follow_redirects=True) + assert response.status_code == 200 + + # Extract UUID from response + url_parts = response.request.path.rsplit("/") + uuid = url_parts[-1] + url_rest = "/".join(url_parts[:-1]) + assert UUID(uuid).version == 4 + assert url_rest == "/digitize/results" + + +def test_digitize_results_post_no_consent(sketch_map_marked, flask_client): + # do not send consent parameter + # -> consent is a checkbox and only send if selected data = {"file": [(BytesIO(sketch_map_marked), "sketch_map.png")]} response = flask_client.post("/digitize/results", data=data, follow_redirects=True) assert response.status_code == 200 @@ -28,6 +43,8 @@ def test_digitize_results_post(sketch_map_marked, flask_client): assert UUID(uuid).version == 4 assert url_rest == "/digitize/results" + # TODO: check consent flag in database + def test_api_status_uuid_sketch_map(uuid_create, flask_client): resp = flask_client.get(f"/api/status/{uuid_create}/sketch-map") diff --git a/tests/unit/test_routes_digitize.py b/tests/unit/test_routes_digitize.py index 0c4e57f5..ddab5c63 100644 --- a/tests/unit/test_routes_digitize.py +++ b/tests/unit/test_routes_digitize.py @@ -37,7 +37,7 @@ def test_digitize_result_get(client): @pytest.mark.skip(reason="Mocking of chained/grouped tasks is too complex for now") def test_digitize_result_post(client, sketch_map_buffer, mock_task): """Redirect to /digitize/results/""" - data = {"file": [(sketch_map_buffer, "sketch_map.png")]} + data = {"file": [(sketch_map_buffer, "sketch_map.png")], "consent": "True"} resp = client.post("/digitize/results", data=data) print(resp.headers.get("Location")) partial_redirect_path = "/digitize/results" From 26ffafd2aa7f0e148c4d395f0e38a7cf8a7989ac Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 5 Jun 2024 14:57:39 +0200 Subject: [PATCH 7/8] WIP: test consent parameter --- tests/integration/test_routes.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/integration/test_routes.py b/tests/integration/test_routes.py index 2328fc5f..80d21fe2 100644 --- a/tests/integration/test_routes.py +++ b/tests/integration/test_routes.py @@ -3,6 +3,9 @@ import pytest +from sketch_map_tool import flask_app as app +from sketch_map_tool.database.client_flask import open_connection + def test_create_results_post(params, flask_client): response = flask_client.post("/create/results", data=params, follow_redirects=True) @@ -42,6 +45,9 @@ def test_digitize_results_post_no_consent(sketch_map_marked, flask_client): url_rest = "/".join(url_parts[:-1]) assert UUID(uuid).version == 4 assert url_rest == "/digitize/results" + with app.app_context(): + raw = select_file(1) + assert raw is False # TODO: check consent flag in database @@ -85,3 +91,14 @@ def test_api_download_uuid_digitize(uuid_digitize, type_, flask_client): def test_health_ok(flask_client): resp = flask_client.get("/api/health") assert resp.status_code == 200 + + +def select_file(id_: int) -> bytes: + """Get an uploaded file stored in the database by ID.""" + query = "SELECT * FROM blob WHERE id = %s" + db_conn = open_connection() + with db_conn.cursor() as curs: + curs.execute(query, [id_]) + raw = curs.fetchone() + if raw: + return raw[3] From 3d727c62382478ea43e07e23f5fc716e905abb34 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Wed, 5 Jun 2024 15:54:47 +0200 Subject: [PATCH 8/8] finish test --- tests/integration/test_routes.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_routes.py b/tests/integration/test_routes.py index 80d21fe2..d59093d7 100644 --- a/tests/integration/test_routes.py +++ b/tests/integration/test_routes.py @@ -1,5 +1,5 @@ from io import BytesIO -from uuid import UUID +from uuid import UUID, uuid4 import pytest @@ -7,6 +7,14 @@ from sketch_map_tool.database.client_flask import open_connection +def get_consent_flag_from_db(file_name: str) -> bool: + query = "SELECT consent FROM blob WHERE file_name = %s" + db_conn = open_connection() + with db_conn.cursor() as curs: + curs.execute(query, [file_name]) + return curs.fetchone()[0] + + def test_create_results_post(params, flask_client): response = flask_client.post("/create/results", data=params, follow_redirects=True) assert response.status_code == 200 @@ -20,7 +28,8 @@ def test_create_results_post(params, flask_client): def test_digitize_results_post(sketch_map_marked, flask_client): - data = {"file": [(BytesIO(sketch_map_marked), "sketch_map.png")], "consent": "True"} + unique_file_name = str(uuid4()) + data = {"file": [(BytesIO(sketch_map_marked), unique_file_name)], "consent": "True"} response = flask_client.post("/digitize/results", data=data, follow_redirects=True) assert response.status_code == 200 @@ -30,12 +39,15 @@ def test_digitize_results_post(sketch_map_marked, flask_client): url_rest = "/".join(url_parts[:-1]) assert UUID(uuid).version == 4 assert url_rest == "/digitize/results" + with app.app_context(): + assert get_consent_flag_from_db(unique_file_name) is True def test_digitize_results_post_no_consent(sketch_map_marked, flask_client): # do not send consent parameter # -> consent is a checkbox and only send if selected - data = {"file": [(BytesIO(sketch_map_marked), "sketch_map.png")]} + unique_file_name = str(uuid4()) + data = {"file": [(BytesIO(sketch_map_marked), unique_file_name)]} response = flask_client.post("/digitize/results", data=data, follow_redirects=True) assert response.status_code == 200 @@ -46,8 +58,7 @@ def test_digitize_results_post_no_consent(sketch_map_marked, flask_client): assert UUID(uuid).version == 4 assert url_rest == "/digitize/results" with app.app_context(): - raw = select_file(1) - assert raw is False + assert get_consent_flag_from_db(unique_file_name) is False # TODO: check consent flag in database @@ -91,14 +102,3 @@ def test_api_download_uuid_digitize(uuid_digitize, type_, flask_client): def test_health_ok(flask_client): resp = flask_client.get("/api/health") assert resp.status_code == 200 - - -def select_file(id_: int) -> bytes: - """Get an uploaded file stored in the database by ID.""" - query = "SELECT * FROM blob WHERE id = %s" - db_conn = open_connection() - with db_conn.cursor() as curs: - curs.execute(query, [id_]) - raw = curs.fetchone() - if raw: - return raw[3]