Use RODARE api instead of hard coded URL:

Its better to use the DOI which always points to the latest version of
the test data repo. This avoids updating the CI at several places each
time there is a new version of the test data repo.

Co-authored-by: David Pape <[email protected]>

.github/workflows/cpu-tests.yml

@@ -176,20 +176,40 @@ jobs:
           # `requirements.txt` and/or extra dependencies are missing in the Docker Conda environment
           diff env_1.yml env_2.yml
 
-      - name: Download test data repository
-        shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
+      - name: Download test data repository from RODARE
+        shell: 'bash -c "docker exec -i mala-cpu python < {0}"'
         run: |
-          # Download test data repository from RODARE. If the version changes
-          # this URL has to be adapted (the number after /record/ and the
-          # version have to be incremented)
-          wget "https://rodare.hzdr.de/record/3004/files/mala-project/test-data-1.8.1.zip"
+          import requests
+
+          # This DOI represents all versions, and will always resolve to the latest one
+          DOI = "https://doi.org/10.14278/rodare.2900"
+
+          # Resolve DOI and get record ID and the associated API URL
+          response = requests.get(DOI)
+          *_, record_id = response.url.split("/")
+          api_url = f"https://rodare.hzdr.de/api/records/{record_id}"
+
+          # Download record from API and get the first file
+          response = requests.get(api_url)
+          record = response.json()
+          size = record["files"][0]["size"]
+          download_link = record["files"][0]["links"]["self"]
+
+          print(size, "bytes", "--", download_link)
+
+          # TODO: implement some sort of auto retry for failed HTTP requests
+          response = requests.get(download_link)
+
+          # Saving Downloaded Content to a File
+          with open("test-data.zip", mode="wb") as file:
+            file.write(response.content)
 
           # Once downloaded, we have to unzip the file. The name of the root
           # folder in the zip file has to be updated for data repository
           # updates as well - the string at the end is the hash of the data
           # repository commit.
-          unzip -q test-data-1.8.1.zip
-          mv mala-project-test-data-741eda6 mala_data
+          #unzip -q test-data-1.8.1.zip
+          #mv mala-project-test-data-741eda6 mala_data
 
       - name: Test mala
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'