Update to v2.10.1

.circleci/config.yml

@@ -13,52 +13,18 @@ defaults:
       ckan -c test-core-circle-ci.ini datastore set-permissions | psql --host=ckan-postgres --username=ckan
       psql --host=ckan-postgres --username=ckan --dbname=${CKAN_DATASTORE_POSTGRES_DB} --command="CREATE extension tablefunc;"
       ckan -c test-core-circle-ci.ini db init
+      gunzip .test_durations.gz
   install_deps: &install_deps
     run: |
       # OS Dependencies
       apt update
-      case $CIRCLE_NODE_INDEX in
-        $NODE_TESTS_CONTAINER)
-        curl -sL https://deb.nodesource.com/setup_10.x | bash -
-        apt install -y nodejs
-        apt install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
-        npm install
-        ;;
-      esac
       apt install -y postgresql-client
   run_tests: &run_tests
     # Tests Backend, split across containers by segments
     run: |
-      mkdir -p ~/junit
-      case $CIRCLE_NODE_INDEX in
-        0) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 1
-        ;;
-        1) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 2
-        ;;
-        2) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 3
-        ;;
-        3) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 4
-        ;;
-      esac
+      mkdir -p ~/junit/result
+      python -m pytest ${PYTEST_COMMON_OPTIONS} --splits 4 --group $((CIRCLE_NODE_INDEX+1)) --splitting-algorithm least_duration
 
-  # Tests Frontend, only in one container
-  start_test_server: &start_test_server
-    run:
-      command: |
-        case $CIRCLE_NODE_INDEX in
-          $NODE_TESTS_CONTAINER) ckan -c test-core-circle-ci.ini run
-          ;;
-        esac
-      background: true
-  run_front_tests: &run_front_tests
-    run:
-      command: |
-        case $CIRCLE_NODE_INDEX in
-          $NODE_TESTS_CONTAINER)
-            sleep 5
-            $(npm bin)/cypress run
-          ;;
-        esac
   ckan_env: &ckan_env
     environment:
       CKAN_DATASTORE_POSTGRES_DB: datastore_test
@@ -70,8 +36,7 @@ defaults:
       CKAN_POSTGRES_USER: ckan_default
       CKAN_POSTGRES_PWD: pass
       PGPASSWORD: ckan
-      NODE_TESTS_CONTAINER: 3
-      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4  --test-group-random-seed 1
+      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=~/junit/result/junit.xml
   pg_image: &pg_image
     image: postgres:10
     environment:
@@ -84,13 +49,13 @@ defaults:
     name: ckan-redis
 
   solr_image: &solr_image
-    image: ckan/ckan-solr:2.9
+    image: ckan/ckan-solr:master
     name: ckan-solr
 
 jobs:
   test-python-3:
     docker:
-      - image: python:3.9-buster
+      - image: python:3.7-bullseye
         <<: *ckan_env
       - <<: *pg_image
       - <<: *redis_image
@@ -104,7 +69,6 @@ jobs:
       - <<: *install_deps
       - run: |
           # Python Dependencies
-          pip install -r requirement-setuptools.txt
           pip install -r requirements.txt
           pip install -r dev-requirements.txt
           python setup.py develop
@@ -115,10 +79,6 @@ jobs:
       - <<: *run_tests
       - store_test_results:
           path: ~/junit
-      - <<: *start_test_server
-      - <<: *run_front_tests
-      - store_artifacts:
-          path: ~/project/cypress/screenshots
       - run: coveralls
 workflows:
   version: 2

.devcontainer/devcontainer.json

@@ -53,7 +53,7 @@
      },
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"onCreateCommand": "pip install --user --upgrade -r requirement-setuptools.txt && pip install --user -r requirements.txt && pip install --user -r dev-requirements.txt",
+	"onCreateCommand": "pip install --user -r requirements.txt && pip install --user -r dev-requirements.txt",
 	"postCreateCommand": "./.devcontainer/setup.sh",
 	"postAttachCommand": "ckan run",
 
@@ -67,4 +67,4 @@
 		  ]
 		}
 	}
-}
+}

.devcontainer/docker-compose.yml

@@ -7,7 +7,7 @@ services:
       dockerfile: .devcontainer/Dockerfile
       args:
         # [Choice] Python version: 3, 3.8, 3.7, 3.6
-        VARIANT: 3.8
+        VARIANT: 3.9
         # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
         NODE_VERSION: "lts/*"
         # On Linux, you may need to update USER_UID and USER_GID below if not your local UID is not 1000.

.devcontainer/setup.sh

@@ -8,9 +8,6 @@ ckan generate config ckan.ini
 mkdir /workspace/data
 ckan config-tool ckan.ini ckan.storage_path=/workspace/data
 
-# Set up Solr URL
-ckan config-tool ckan.ini solr_url=http://localhost:8983/solr/ckan
-
 # Set up site URL
 ckan config-tool ckan.ini ckan.site_url=https://$CODESPACE_NAME-5000.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN
 
@@ -28,7 +25,5 @@ ckan config-tool ckan.ini \
     ckan.datastore.read_url=postgresql://datastore_default:pass@localhost/datastore_default \
     ckan.datapusher.url=http://localhost:8800 \
     ckan.datapusher.callback_url_base=http://localhost:5000 \
-    "ckan.plugins=datastore datapusher datatables_view" \
-    "ckan.views.default_views= image_view text_view datatables_view"
-
+    "ckan.plugins=activity datastore datapusher datatables_view"
 ckan datastore set-permissions | psql $(grep ckan.datastore.write_url ckan.ini | awk -F= '{print $2}')

.flake8

@@ -0,0 +1,65 @@
+[flake8]
+exclude =
+        .*
+        ./contrib
+        ./bin
+per-file-ignores =
+                 test_*.py:E501
+extend-exclude =
+               ckan/__init__.py,ckan/config/middleware.py
+               ckan/lib/app_globals.py
+               ckan/lib/cli.py
+               ckan/lib/create_test_data.py
+               ckan/lib/dictization/__init__.py
+               ckan/lib/dictization/model_dictize.py
+               ckan/lib/dictization/model_save.py
+               ckan/lib/email_notifications.py
+               ckan/lib/hash.py
+               ckan/lib/jinja_extensions.py
+               ckan/lib/maintain.py
+               ckan/lib/navl/validators.py
+               ckan/lib/plugins.py
+               ckan/lib/search/__init__.py
+               ckan/lib/search/index.py
+               ckan/lib/search/query.py
+               ckan/logic/action/__init__.py
+               ckan/logic/action/delete.py
+               ckan/logic/action/get.py
+               ckan/logic/action/update.py
+               ckan/logic/auth/create.py
+               ckan/logic/auth/delete.py
+               ckan/logic/auth/get.py
+               ckan/logic/auth/update.py
+               ckan/logic/converters.py
+               ckan/logic/validators.py
+               ckan/model/__init__.py
+               ckan/model/dashboard.py
+               ckan/model/domain_object.py
+               ckan/model/follower.py
+               ckan/model/group.py
+               ckan/model/group_extra.py
+               ckan/model/license.py
+               ckan/model/meta.py
+               ckan/model/misc.py
+               ckan/model/modification.py
+               ckan/model/package.py
+               ckan/model/package_extra.py
+               ckan/model/package_relationship.py
+               ckan/model/resource.py
+               ckan/model/system_info.py
+               ckan/model/tag.py
+               ckan/model/task_status.py
+               ckan/model/term_translation.py
+               ckan/model/tracking.py
+               ckan/model/user.py
+               ckan/model/vocabulary.py
+               ckan/authz.py
+               ckanext/datastore/logic/action.py
+               ckanext/datastore/tests/test_create.py
+               ckanext/example_idatasetform/plugin.py
+               ckanext/example_itemplatehelpers/plugin.py
+               ckanext/multilingual/plugin.py
+               ckanext/stats/stats.py
+               ckanext/test_tag_vocab_plugin.py
+               doc/conf.py
+               setup.py

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,16 +7,17 @@ assignees: ''
 
 ---
 
-**CKAN version**
+## CKAN version
 
-**Describe the bug**
+## Describe the bug
 A clear and concise description of what the bug is.
 
-**Steps to reproduce**
+### Steps to reproduce
 Steps to reproduce the behavior:
 
-**Expected behavior**
+### Expected behavior
 A clear and concise description of what you expected to happen.
 
-**Additional details**
+### Additional details
 If possible, please provide the full stack trace of the error raised,  or add screenshots to help explain your problem.
+

.github/ISSUE_TEMPLATE/config.yml

@@ -9,6 +9,6 @@ contact_links:
   - name: Security Issues
     url: mailto:[email protected]
     about: Please report any security related vulnerabilities here.
-  - name: Ideas Repository
-    url: https://github.com/ckan/ideas
-    about: For new feature requests or discussion, please create an issue on the ideas repository.
+  - name: Ideas and Discussion
+    url: https://github.com/ckan/ckan/discussions
+    about: For new feature requests or discussion, please create use the discussion forum.

.github/codeql/codeql-config.yml

@@ -0,0 +1,4 @@
+name: "CKAN CodeQL config"
+
+paths-ignore:
+  - '**/vendor/**'

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 3 * * 2'
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    permissions:
+      actions: read  # for github/codeql-action/init to get workflow details
+      contents: read  # for actions/checkout to fetch code
+      security-events: write  # for github/codeql-action/autobuild to send a status report
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['python', 'javascript']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        config-file: ./.github/codeql/codeql-config.yml
+        setup-python-dependencies: false
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/cypress.yml

@@ -0,0 +1,73 @@
+name: Cypress
+on: [pull_request]
+env:
+  NODE_VERSION: '16'
+  PYTHON_VERSION: '3.7'
+
+permissions:
+  contents: read
+
+jobs:
+  cypress:
+    runs-on: ubuntu-latest
+    services:
+      ckan-postgres:
+        image: postgres:10
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          POSTGRES_USER: ckan_default
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DB: ckan_test
+
+      ckan-redis:
+        image: redis
+        ports:
+          - 6379:6379
+      ckan-solr:
+        image: ckan/ckan-solr:master
+        ports:
+          - 8983:8983
+
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@localhost/ckan_test
+      CKAN_SOLR_URL: http://localhost:8983/solr/ckan
+      CKAN_REDIS_URL: redis://localhost:6379/1
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install python deps
+        run: pip install -r requirements.txt -r dev-requirements.txt -e.
+
+      - name: Init environment
+        run: |
+          ckan -c test-core-cypress.ini db init
+
+      - name: Run Cypress
+        uses: cypress-io/github-action@v2
+        with:
+          start: ckan -c test-core-cypress.ini run
+
+      - uses: actions/upload-artifact@v1
+        if: failure()
+        with:
+          name: cypress-screenshots
+          path: cypress/screenshots
+      # Test run video was always captured, so this action uses "always()" condition
+      - uses: actions/upload-artifact@v1
+        if: always()
+        with:
+          name: cypress-videos
+          path: cypress/videos