Merge development into main

.github/workflows/build_and_deploy.yml

@@ -59,7 +59,9 @@ jobs:
           context: src/backend
           target: prod
           push: true
-          tags: "ghcr.io/hotosm/fmtm/backend:${{ env.API_VERSION }}-${{ github.ref_name }}"
+          tags: |
+            "ghcr.io/hotosm/fmtm/backend:${{ env.API_VERSION }}-${{ github.ref_name }}"
+            "ghcr.io/hotosm/fmtm/backend:latest"
           build-args: |
             APP_VERSION=${{ env.API_VERSION }}
 

.github/workflows/frontend_test.yml

@@ -0,0 +1,28 @@
+name: Frontend Tests
+
+on:
+  push:
+    paths:
+      - "src/frontend/**" # Trigger the workflow only when files within srv/frontend change
+      - ".github/workflows/**" # Also trigger if github workflow changes
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Run Frontend Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: "18" # Change this to your preferred Node.js version
+
+      - name: Test Frontend Main
+        run: |
+          cd src/frontend/main
+          npm install
+          npm run test

.github/workflows/pytest.yml

@@ -14,29 +14,32 @@ on:
   # Allow manual trigger (workflow_dispatch)
   workflow_dispatch:
 
-env:
-  # using fmtm-db compose name fails as running FMTM outside docker, port map to localhost service instead
-  FMTM_DB_URL: postgresql+psycopg2://fmtm:fmtm@localhost:5432/fmtm
-  ODK_CENTRAL_URL: http://localhost:8383
-  ODK_CENTRAL_USER: odk
-  ODK_CENTRAL_PASSWD: odk
-  OSM_CLIENT_ID: test
-  OSM_CLIENT_SECRET: test
-  OSM_URL: https://www.openstreetmap.org
-  OSM_SCOPE: read_prefs
-  OSM_LOGIN_REDIRECT_URI: http://127.0.0.1:8000/auth/callback/
-  OSM_SECRET_KEY: test
-  URL_SCHEME: "http"
-  FRONTEND_MAIN_URL: "localhost:8080"
-  FRONTEND_MAP_URL: "localhost:8081"
-
 permissions:
   contents: read
 
 jobs:
   test:
     runs-on: ubuntu-latest
-    # pytest needs a valid reachable database service.
+
+    container:
+      image: ghcr.io/hotosm/fmtm/backend:ci
+      env:
+        FMTM_DB_URL: postgresql+psycopg2://fmtm:fmtm@fmtm-db:5432/fmtm
+        ODK_CENTRAL_URL: http://localhost:8383
+        ODK_CENTRAL_USER: odk
+        ODK_CENTRAL_PASSWD: odk
+        OSM_CLIENT_ID: test
+        OSM_CLIENT_SECRET: test
+        OSM_URL: https://www.openstreetmap.org
+        OSM_SCOPE: read_prefs
+        OSM_LOGIN_REDIRECT_URI: http://127.0.0.1:8000/auth/callback/
+        OSM_SECRET_KEY: test
+        URL_SCHEME: "http"
+        FRONTEND_MAIN_URL: "localhost:8080"
+        FRONTEND_MAP_URL: "localhost:8081"
+      options: --user root
+
+    # pytest needs a database
     services:
       # Label used to access the postgres service container
       fmtm-db:
@@ -47,36 +50,16 @@ jobs:
           POSTGRES_PASSWORD: fmtm
           POSTGRES_DB: fmtm
           POSTGRES_USER: fmtm
-        ports:
-          # Opens tcp port 5432 on the host and service container,
-          # since we running pytest on actions machine, not in another service container
-          - 5432:5432
         # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v3
-        with:
-          python-version: "3.10"
-      - uses: actions/cache@v3
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: Install dependencies
-        run: |
-          cd src/backend
-          python -m pip install pdm==2.6.1
-          pdm export --dev > requirements-all.txt
-          pip install -r requirements-all.txt
-      - name: Run pytest
+      - name: Run pytest as appuser
         run: |
           cd src/backend
-          pytest
+          gosu appuser pytest

docker-compose.josm.yml

@@ -0,0 +1,54 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+version: "3"
+
+networks:
+  x11:
+
+services:
+  josm:
+    image: "ghcr.io/hotosm/fmtm/josm:latest"
+    build:
+      context: josm
+    container_name: josm
+    environment:
+      - DISPLAY=josm-novnc:0.0
+    depends_on:
+      - josm-novnc
+    networks:
+      - fmtm-dev
+      - x11
+    ports:
+      - 8111:80
+    restart: unless-stopped
+
+  josm-novnc:
+    image: "ghcr.io/hotosm/fmtm/josm-novnc:latest"
+    build:
+      context: josm/novnc
+    container_name: josm_novnc
+    environment:
+      - DISPLAY_WIDTH=1280
+      - DISPLAY_HEIGHT=600
+      - RUN_XTERM=no
+      - RUN_FLUXBOX=no
+    ports:
+      - "8112:8080"
+    networks:
+      - x11
+    restart: unless-stopped

docker-compose.yml

@@ -52,6 +52,7 @@ services:
     volumes:
       - fmtm_images:/opt/app/images
       - ./src/backend/app:/opt/app
+      # - ../osm-fieldwork/osm_fieldwork:/home/appuser/.local/lib/python3.10/site-packages/osm_fieldwork
     depends_on:
       - fmtm-db
       - central-proxy

docs/DEV-2.-Backend.md

@@ -126,6 +126,52 @@ Example launch.json config for vscode:
 
 > Note: either port 5678 needs to be bound to your localhost (default), or the `host` parameter can be set to the container IP address.
 
+## Debugging osm-fieldwork
+
+`osm-fieldwork` is an integral package for much of the functionality in FMTM.
+
+Creating a new release during development may not always be feasible.
+
+**Via Dockerfile**
+
+- The debug stages in the backend Dockerfile install the latest osm-fieldwork repo `main` branch.
+- To re-build, run: `docker compose build api --no-cache`.
+
+> Note: this is useful to debug functionality not yet released in a stable version on PyPi.
+
+**Via Bind-Mount**
+
+- Alternatively, a development version of osm-fieldwork can be mounted into the FMTM container.
+- Clone the osm-fieldwork repo to the same root directory as FMTM.
+- Uncomment the line in docker-compose.yml
+
+```yaml
+- ../osm-fieldwork/osm_fieldwork:/home/appuser/.local/lib/python3.10/site-packages/osm_fieldwork
+```
+
+- Run the docker container with your local version of osm-fieldwork.
+- Code changes to osm-fieldwork should be reflected immediately. If they are not, run:
+  `docker compose restart api`.
+
+> Note: this is useful for debugging features during active development.
+
+## Running JOSM in the dev stack
+
+- Run JOSM with FMTM:
+
+```bash
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.josm.yml \
+  up -d
+```
+
+This adds JOSM to the docker compose stack for local development.
+Access the JOSM Remote API: <http://localhost:8111>
+Access the JOSM GUI in browser: <http://localhost:8112>
+
+You can now call the JOSM API from FMTM and changes will be reflected in the GUI.
+
 ## Conclusion
 
 Running the FMTM project is easy with Docker. You can also run the

josm/Dockerfile

@@ -0,0 +1,78 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+FROM docker.io/debian:bookworm as base
+ARG [email protected]
+LABEL org.hotosm.fmtm.maintainer="${MAINTAINER}" \
+      org.hotosm.fmtm.josm-port="8111" \
+      org.hotosm.fmtm.nginx-port="80"
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends "locales" "ca-certificates" \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && update-ca-certificates
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+
+
+FROM base as gpg-key
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "curl" \
+        "gnupg2" \
+    && rm -rf /var/lib/apt/lists/*
+RUN curl -sS https://josm.openstreetmap.de/josm-apt.key \
+    | gpg --dearmor | tee /opt/josm.gpg
+
+
+
+FROM base as runtime
+COPY --from=gpg-key \
+    /opt/josm.gpg /etc/apt/trusted.gpg.d/josm.gpg
+RUN echo \
+    "deb [arch=$(dpkg --print-architecture) \
+    signed-by=/etc/apt/trusted.gpg.d/josm.gpg] \
+    https://josm.openstreetmap.de/apt alldist universe" \ 
+    | tee /etc/apt/sources.list.d/josm.list > /dev/null
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "gosu" \
+        "josm" \
+        "nginx" \
+    && rm -rf /var/lib/apt/lists/*
+COPY container-entrypoint.sh /container-entrypoint.sh
+# Add non-root user
+RUN useradd --system -r -u 101 -m -c "nginx user" \
+    -d /home/nginx -s /bin/false nginx \
+    && chmod +x /container-entrypoint.sh
+COPY --chown=nginx \
+    preferences.xml /home/nginx/.config/JOSM/preferences.xml
+# Replace default nginx config
+COPY --chown=nginx \
+    nginx-josm.conf /etc/nginx/sites-enabled/default
+# Run as root, change to nginx user in entrypoint
+ENTRYPOINT ["/container-entrypoint.sh"]

josm/container-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Start JOSM as nginx (unpriv) user
+gosu nginx josm &
+
+exec nginx -g "daemon off;"

josm/nginx-josm.conf

@@ -0,0 +1,13 @@
+server {
+    listen 80 default_server;
+    server_name _;
+
+    location / {
+        proxy_pass http://127.0.0.1:8111;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }    
+}

josm/novnc/Dockerfile

@@ -0,0 +1,6 @@
+FROM docker.io/theasp/novnc:latest
+COPY index.html /usr/share/novnc/
+RUN useradd -r -u 900 -m -c "novnc account" -d /home/appuser -s /bin/false appuser \
+    && chown -R appuser:appuser /app
+WORKDIR /app
+USER appuser

josm/novnc/index.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <meta http-equiv="refresh" content="0; URL=vnc.html?resize=scale" />
+  </head>
+  <body>
+    <p>If you see this <a href="vnc.html?resize=scale">click here</a>.</p>
+  </body>
+</html>

josm/preferences.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<preferences xmlns='http://josm.openstreetmap.de/preferences-1.0' version='18789'>
+  <tag key='josm.version' value='18789'/>
+  <tag key='draw.splashscreen' value='false'/>
+  <tag key='expert' value='true'/>
+  <tag key='gui.start.animation' value='false'/>
+  <tag key='cache.capabilities1637351842' value='1691490836'/>
+  <tag key='cache.motd.html' value='1691490835'/>
+  <tag key='cache.motd.html.java' value='11.0.18'/>
+  <tag key='cache.motd.html.lang' value='En:'/>
+  <tag key='cache.motd.html.version' value='18789'/>
+  <tag key='draw.rawgps.colormode' value='0'/>
+  <tag key='draw.rawgps.lines' value='2'/>
+  <tag key='mappaint.renderer-class-name' value='org.openstreetmap.josm.data.osm.visitor.paint.StyledMapRenderer'/>
+  <tag key='oauth.use-for-all-requests' value='false'/>
+  <tag key='org.openstreetmap.josm.gui.preferences.PreferenceDialog.geometry' value='x=1610,y=585,width=800,height=774'/>
+  <tag key='preferences.reset.draw.rawgps.lines' value='true'/>
+  <list key='projection.sub.core:mercator'>
+  </list>
+  <tag key='propertiesdialog.preview-on-hover' value='false'/>
+  <tag key='proxy.policy' value='no-proxy'/>
+  <tag key='remotecontrol.enabled' value='true'/>
+  <list key='validator.skip'>
+  </list>
+  <list key='validator.skipBeforeUpload'>
+  </list>
+</preferences>