Enable opt-out of the usage of uploaded sketch maps for the improvement of the tool

client-src/digitize/index.js

@@ -11,6 +11,8 @@ filebokz();
 // disable submit...
 setDisabled("submitBtn", true);
 
+const consentCheckbox = document.getElementById("consent");
+
 // ...until files are added
 const fileElement = document.querySelector(".filebokz");
 fileElement.addEventListener("file-added", () => {

scripts/flask.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 # Run Flask in debug mode
-poetry run flask --app sketch_map_tool/routes.py --debug run --port 8081
+pybabel compile -d sketch_map_tool/translations
+flask --app sketch_map_tool/routes.py --debug run --port 8081

sketch_map_tool/database/client_flask.py

@@ -85,23 +85,33 @@ def set_async_result_ids(request_uuid, map_: dict[REQUEST_TYPES, str]):
     _insert_id_map(request_uuid, map_)
 
 
-def insert_files(files) -> list[int]:
+def insert_files(files, consent: bool) -> list[int]:
     """Insert uploaded files as blob into the database and return primary keys"""
     create_query = """
     CREATE TABLE IF NOT EXISTS blob(
         id SERIAL PRIMARY KEY,
         file_name VARCHAR,
         file BYTEA,
-        ts timestamp with time zone DEFAULT now()
+        consent BOOLEAN,
+        ts TIMESTAMP WITH TIME ZONE DEFAULT now()
         )
-        """
-    insert_query = "INSERT INTO blob(file_name, file) VALUES (%s, %s) RETURNING id"
+    """
+    insert_query = (
+        "INSERT INTO blob(file_name, file, consent) VALUES (%s, %s, %s) RETURNING id"
+    )
     db_conn = open_connection()
     with db_conn.cursor() as curs:
         curs.execute(create_query)
         ids = []
         for file in files:
-            curs.execute(insert_query, (secure_filename(file.filename), file.read()))
+            curs.execute(
+                insert_query,
+                (
+                    secure_filename(file.filename),
+                    file.read(),
+                    consent,
+                ),
+            )
             ids.append(curs.fetchone()[0])
     return ids
 

sketch_map_tool/map_generation/generate_pdf.py

@@ -26,7 +26,7 @@
 Image.MAX_IMAGE_PIXELS = None
 
 
-def generate_pdf(  # noqa: C901
+def generate_pdf(
     map_image_input: PILImage,
     qr_code: Drawing,
     format_: PaperFormat,

sketch_map_tool/routes.py

@@ -121,12 +121,17 @@ def digitize(lang="en") -> str:
 @app.post("/<lang>/digitize/results")
 def digitize_results_post(lang="en") -> Response:
     """Upload files to create geodata results"""
+    # "consent" is a checkbox and value is only send if it is checked
+    if "consent" in request.form.keys():
+        consent: bool = True
+    else:
+        consent: bool = False
     # No files uploaded
     if "file" not in request.files:
         return redirect(url_for("digitize", lang=lang))
     files = request.files.getlist("file")
     validate_uploaded_sketchmaps(files)
-    ids = db_client_flask.insert_files(files)
+    ids = db_client_flask.insert_files(files, consent)
     file_names = [db_client_flask.select_file_name(i) for i in ids]
     args = [
         upload_processing.read_qr_code(to_array(db_client_flask.select_file(_id)))

sketch_map_tool/templates/digitize.html

@@ -48,6 +48,7 @@
             </div>
             <div>
                 <p>
+                    <input type="checkbox" id="consent" name="consent" value="True" checked>
                     {{ _('By uploading the file(s), you agree that the content of the files and the time of upload will
                     be saved and used for the further development of the service.') }}
                 </p>

tests/fixtures/cassette/test_generate_quality_report

@@ -4204,4 +4204,108 @@ interactions:
     status:
       code: 200
       message: OK
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: https://maps.heigit.org/sketch-map-tool/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fjpeg&TRANSPARENT=FALSE&LAYERS=world_imagery&WIDTH=1716&HEIGHT=1436&SRS=EPSG%3A3857&STYLES=&BBOX=964445.3646475708%2C6343463.48326091%2C967408.255014792%2C6345943.466874749
+  response:
+    body:
+      string: '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+
+        <html><head>
+
+        <title>403 Forbidden</title>
+
+        </head><body>
+
+        <h1>Forbidden</h1>
+
+        <p>You don''t have permission to access this resource.</p>
+
+        <hr>
+
+        <address>Apache/2.4.52 (Ubuntu) Server at maps.heigit.org Port 443</address>
+
+        </body></html>
+
+        '
+    headers:
+      Connection:
+      - Keep-Alive
+      Content-Length:
+      - '281'
+      Content-Type:
+      - text/html; charset=iso-8859-1
+      Date:
+      - Mon, 03 Jun 2024 13:06:07 GMT
+      Keep-Alive:
+      - timeout=5, max=100
+      Server:
+      - Apache/2.4.52 (Ubuntu)
+      Strict-Transport-Security:
+      - max-age=63072000; includeSubdomains;
+    status:
+      code: 403
+      message: Forbidden
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: https://maps.heigit.org/sketch-map-tool/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=FALSE&LAYERS=world_imagery_fallback&WIDTH=1716&HEIGHT=1436&SRS=EPSG%3A3857&STYLES=&BBOX=964445.3646475708%2C6343463.48326091%2C967408.255014792%2C6345943.466874749
+  response:
+    body:
+      string: '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+
+        <html><head>
+
+        <title>403 Forbidden</title>
+
+        </head><body>
+
+        <h1>Forbidden</h1>
+
+        <p>You don''t have permission to access this resource.</p>
+
+        <hr>
+
+        <address>Apache/2.4.52 (Ubuntu) Server at maps.heigit.org Port 443</address>
+
+        </body></html>
+
+        '
+    headers:
+      Connection:
+      - Keep-Alive
+      Content-Length:
+      - '281'
+      Content-Type:
+      - text/html; charset=iso-8859-1
+      Date:
+      - Mon, 03 Jun 2024 13:06:08 GMT
+      Keep-Alive:
+      - timeout=5, max=100
+      Server:
+      - Apache/2.4.52 (Ubuntu)
+      Strict-Transport-Security:
+      - max-age=63072000; includeSubdomains;
+    status:
+      code: 403
+      message: Forbidden
 version: 1

tests/fixtures/cassette/test_generate_sketch_map

@@ -19947,8 +19947,6 @@ interactions:
       - '"1871091362"'
       Strict-Transport-Security:
       - max-age=63072000
-      Transfer-Encoding:
-      - chunked
       Vary:
       - Accept-Encoding
       Via:
@@ -19970,4 +19968,48 @@ interactions:
     status:
       code: 200
       message: OK
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: https://maps.heigit.org/osm-carto/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=FALSE&LAYERS=heigit%3Aosm-carto%402xx&WIDTH=1716&HEIGHT=1436&SRS=EPSG%3A3857&STYLES=&BBOX=964445.3646475708%2C6343463.48326091%2C967408.255014792%2C6345943.466874749
+  response:
+    body:
+      string: !!binary |
+        iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhV
+        AAAAAElFTkSuQmCC
+    headers:
+      Access-control-allow-origin:
+      - '*'
+      Cache-Control:
+      - max-age=86400
+      Connection:
+      - Upgrade, Keep-Alive
+      Content-length:
+      - '759648'
+      Content-type:
+      - image/png
+      Date:
+      - Tue, 04 Jun 2024 15:02:02 GMT
+      Expires:
+      - Wed, 05 Jun 2024 15:02:02 GMT
+      Keep-Alive:
+      - timeout=5, max=100
+      Server:
+      - Apache/2.4.52 (Ubuntu)
+      Strict-Transport-Security:
+      - max-age=63072000; includeSubdomains;
+      Upgrade:
+      - h2
+    status:
+      code: 200
+      message: OK
 version: 1

tests/integration/test_database_client_flask.py

@@ -29,7 +29,7 @@ def file_ids(files, flask_app):
     """IDs of uploaded files stored in the database."""
     with flask_app.app_context():
         # setup
-        ids = db_client_flask.insert_files(files)
+        ids = db_client_flask.insert_files(files, consent=True)
         yield ids
         # teardown
         for i in ids:
@@ -80,7 +80,7 @@ def test_get_async_result_id(flask_app, uuid):
 
 def test_insert_files(flask_app, files):
     with flask_app.app_context():
-        ids = client_flask.insert_files(files)
+        ids = client_flask.insert_files(files, consent=True)
         try:
             assert len(ids) == 2
             assert isinstance(ids[0], int)
@@ -92,7 +92,7 @@ def test_insert_files(flask_app, files):
 
 def test_delete_file(flask_app, files):
     with flask_app.app_context():
-        ids = client_flask.insert_files(files)
+        ids = client_flask.insert_files(files, consent=True)
         for i in ids:
             # No error should be raised
             client_flask.delete_file(i)

tests/integration/test_routes.py

@@ -1,8 +1,19 @@
 from io import BytesIO
-from uuid import UUID
+from uuid import UUID, uuid4
 
 import pytest
 
+from sketch_map_tool import flask_app as app
+from sketch_map_tool.database.client_flask import open_connection
+
+
+def get_consent_flag_from_db(file_name: str) -> bool:
+    query = "SELECT consent FROM blob WHERE file_name = %s"
+    db_conn = open_connection()
+    with db_conn.cursor() as curs:
+        curs.execute(query, [file_name])
+        return curs.fetchone()[0]
+
 
 def test_create_results_post(params, flask_client):
     response = flask_client.post("/create/results", data=params, follow_redirects=True)
@@ -17,7 +28,8 @@ def test_create_results_post(params, flask_client):
 
 
 def test_digitize_results_post(sketch_map_marked, flask_client):
-    data = {"file": [(BytesIO(sketch_map_marked), "sketch_map.png")]}
+    unique_file_name = str(uuid4())
+    data = {"file": [(BytesIO(sketch_map_marked), unique_file_name)], "consent": "True"}
     response = flask_client.post("/digitize/results", data=data, follow_redirects=True)
     assert response.status_code == 200
 
@@ -27,6 +39,28 @@ def test_digitize_results_post(sketch_map_marked, flask_client):
     url_rest = "/".join(url_parts[:-1])
     assert UUID(uuid).version == 4
     assert url_rest == "/digitize/results"
+    with app.app_context():
+        assert get_consent_flag_from_db(unique_file_name) is True
+
+
+def test_digitize_results_post_no_consent(sketch_map_marked, flask_client):
+    # do not send consent parameter
+    # -> consent is a checkbox and only send if selected
+    unique_file_name = str(uuid4())
+    data = {"file": [(BytesIO(sketch_map_marked), unique_file_name)]}
+    response = flask_client.post("/digitize/results", data=data, follow_redirects=True)
+    assert response.status_code == 200
+
+    # Extract UUID from response
+    url_parts = response.request.path.rsplit("/")
+    uuid = url_parts[-1]
+    url_rest = "/".join(url_parts[:-1])
+    assert UUID(uuid).version == 4
+    assert url_rest == "/digitize/results"
+    with app.app_context():
+        assert get_consent_flag_from_db(unique_file_name) is False
+
+    # TODO: check consent flag in database
 
 
 def test_api_status_uuid_sketch_map(uuid_create, flask_client):

tests/unit/test_routes_digitize.py

@@ -37,7 +37,7 @@ def test_digitize_result_get(client):
 @pytest.mark.skip(reason="Mocking of chained/grouped tasks is too complex for now")
 def test_digitize_result_post(client, sketch_map_buffer, mock_task):
     """Redirect to /digitize/results/<uuid>"""
-    data = {"file": [(sketch_map_buffer, "sketch_map.png")]}
+    data = {"file": [(sketch_map_buffer, "sketch_map.png")], "consent": "True"}
     resp = client.post("/digitize/results", data=data)
     print(resp.headers.get("Location"))
     partial_redirect_path = "/digitize/results"