Governance Actions and DRep Metadata fix

.github/workflows/build-and-deploy-beta.yml

@@ -27,6 +27,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"
@@ -37,6 +38,7 @@ jobs:
       SENTRY_IGNORE_API_RESOLUTION_ERROR: "1"
       TRAEFIK_LE_EMAIL: "[email protected]"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
+      IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: "false"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/build-and-deploy-dev.yml

@@ -3,6 +3,15 @@ run-name: Deploy by @${{ github.actor }}
 
 on:
   workflow_dispatch:
+    inputs:
+      isProposalDiscussionForumEnabled:
+        description: "Enable proposal discussion forum"
+        required: true
+        type: choice
+        default: "disabled"
+        options:
+          - "enabled"
+          - "disabled"
 
 env:
   ENVIRONMENT: "dev"
@@ -26,6 +35,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"
@@ -37,6 +47,7 @@ jobs:
       SENTRY_IGNORE_API_RESOLUTION_ERROR: "1"
       TRAEFIK_LE_EMAIL: "[email protected]"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
+      IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/build-and-deploy-staging.yml

@@ -28,6 +28,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"
@@ -39,6 +40,7 @@ jobs:
       SENTRY_IGNORE_API_RESOLUTION_ERROR: "1"
       TRAEFIK_LE_EMAIL: "[email protected]"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
+      IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: "false"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/build-and-deploy-test-stack.yml

@@ -20,6 +20,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }}
       PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
@@ -46,4 +47,4 @@ jobs:
           options: |
             --verbose
         env:
-          GOVTOOL_TAG: ${{ github.sha }}
+          GOVTOOL_TAG: ${{ github.sha }}

.github/workflows/build-and-deploy-test.yml

@@ -28,6 +28,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"
@@ -39,6 +40,7 @@ jobs:
       SENTRY_IGNORE_API_RESOLUTION_ERROR: "1"
       TRAEFIK_LE_EMAIL: "[email protected]"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
+      IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: "false"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/code_check_frontend.yml

@@ -4,7 +4,7 @@ on:
   push:
     paths:
       - govtool/frontend/**
-      - .github/workflows/test_frontend.yml
+      - .github/workflows/code_check_frontend.yml
 
 defaults:
   run:
@@ -26,13 +26,17 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: "govtool/frontend/.nvmrc"
+          registry-url: "https://registry.npmjs.org/"
+          node-version-file: "./govtool/frontend/.nvmrc"
+          scope: "@intersect.mbo"
+      - run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPMRC_TOKEN }}
 
       - name: 🧪 Test
         env:
-          NODE_OPTIONS: "--max_old_space_size=4096"
+          NODE_OPTIONS: "--max_old_space_size=8192"
         run: |
-          npm install
           npm run test
 
   lint:
@@ -50,11 +54,15 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: "govtool/frontend/.nvmrc"
+          registry-url: "https://registry.npmjs.org/"
+          node-version-file: "./govtool/frontend/.nvmrc"
+          scope: "@intersect.mbo"
+      - run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPMRC_TOKEN }}
 
       - name: 👕 Lint
         run: |
-          npm install
           npm run lint
 
   type_check:
@@ -72,9 +80,13 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: "govtool/frontend/.nvmrc"
+          registry-url: "https://registry.npmjs.org/"
+          node-version-file: "./govtool/frontend/.nvmrc"
+          scope: "@intersect.mbo"
+      - run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPMRC_TOKEN }}
 
       - name: 🔍 Type Check
         run: |
-          npm install
           npm run tsc

.github/workflows/deploy-dev-from-tag.yml

@@ -28,6 +28,7 @@ jobs:
       SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }}
       TRAEFIK_LE_EMAIL: "[email protected]"
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }}
       PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}

.github/workflows/resync-cardano-node-and-db-sync.yml

@@ -36,6 +36,7 @@ jobs:
       SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }}
       TRAEFIK_LE_EMAIL: "[email protected]"
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }}
       PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}

CHANGELOG.md

@@ -24,6 +24,8 @@ changes.
 
 ### Added
 
+- added `bio` `dRepName` `email` `references` `metadataValid` and `metadataStatus` fields to `drep/list`
+- added `metadatavalidationmaxconcurrentrequests` field to the backend config
 - added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876)
 - added pagination to `drep/list` [Issue 756](https://github.com/IntersectMBO/govtool/issues/756)
 - added search query param to the `drep/getVotes` [Issue 640](https://github.com/IntersectMBO/govtool/issues/640)
@@ -60,6 +62,7 @@ changes.
 - Add type check & lint to github actions [Issue 512](https://github.com/IntersectMBO/govtool/issues/512)
 - Add eslint & prettier to frontend package [Issue 166](https://github.com/IntersectMBO/govtool/issues/166)
 - Add DRep list pagination [Issue 740](https://github.com/IntersectMBO/govtool/issues/740)
+- Add PDF pillar [Issue 1090](https://github.com/IntersectMBO/govtool/issues/1090)
 
 ### Fixed
 
@@ -92,6 +95,7 @@ changes.
 
 ### Changed
 
+- `proposal.about` changed to `proposal.abstract`
 - `drep/info` now returns 4 different tx hashes instead of one latest tx hash [Issue 688](https://github.com/IntersectMBO/govtool/issues/688)
 - `proposal/list` allows user to search by tx hash [Issue 603](https://github.com/IntersectMBO/govtool/issues/603)
 - `proposal/list` returns additional data such ass `expiryEpochNo`, `createdEpochNo`, `title`, `about`, `motivation`,

README.md

@@ -28,7 +28,8 @@ Learn more; [docs.sanchogov.tools](https://docs.sanchogov.tools/).
 
 - [GovTool Backend](./govtool/backend/README.md)
 - [GovTool Frontend](./govtool/frontend/README.md)
-- [GovTool Infrastructure](./infra/)
+- [GovTool Infrastructure](./infra/terraform/README.md)
+- [GovTool deployment process](./scripts/govtool/README.md)
 - [Documentation](./docs/)
 - [Tests](./tests/)
 

docs/architecture/README.md

@@ -0,0 +1,110 @@
+# GovTool Software Architecture Documentation
+
+**Valid as of: 2024-04-30**
+
+## Overview
+
+This documentation outlines the software architecture of the GovTool project.
+GovTool is the decentralized application composed of a React frontend,
+Haskell backend connected to db-sync, and a Node.js
+utility backend that serves validation of the governance actions metadata.
+
+## Frontend
+
+# Technology Stack
+
+- [React](https://react.dev/): The library for web user interfaces.
+- [Vite](https://vitejs.dev/): The build tool for modern web development.
+- [TypeScript](https://www.typescriptlang.org/): The language for static typing.
+- [@mui](https://mui.com/): The library for React components.
+- [react-query](https://tanstack.com/query/latest): The library for data fetching.
+- [react-router](https://reactrouter.com/): The library for routing.
+- [cardano-serialization-lib](https://github.com/Emurgo/cardano-serialization-lib): The library for serialization and deserialization of Cardano data structures.
+
+# Description
+
+Frontend is a React application using Vite as a built tool to enhance development speed and optimize production builds. Frontend interacts with the backend service via REST API and with the Cardano blockchain via cardano-serialization-lib and connected supported wallets (for the list of compatible wallets go [here](https://docs.sanchogov.tools/how-to-use-the-govtool/getting-started/get-a-compatible-wallet)).
+
+# Components
+
+- **Direct voter** - direct voter is a DRep which does not have a metadata and have all the ADA delegated to themselves. This component combines the registration and delegation process into one step. Direct voters are not visible in the DRep directory.
+  Direct voters ui components are (no specific file names are provided as they might be continuosly updated):
+
+  - Direct voter registration card - UI component visible on the dashboard allowing to navigate to the Direct voter registration form. It displays current Direct voter status and amount of ADA.
+  - Direct voter registration form - UI component allowing to register as a Direct voter and delegate all the ADA to themselves. Under the hood metadata anchor is mocked with provided default values.
+    Direct voter uses following CSL services:
+  - TransactionBuilder - to build the transaction
+  - CertificatesBuilder - to build the delegation certificate
+  - DrepRegistration - to build the DRep registration certificate
+
+- **DRep** - DRep is a Decentralized Representative which has a metadata and can delegate ADA to other DReps. DRep registration and delegation are separate processes. DReps are visible in the DRep directory.
+
+  DRep ui components are (no specific file names are provided as they might be continuosly updated):
+
+  - DRep registration card - UI component visible on the dashboard allowing to navigate to the DRep registration form. It displays current DRep status and amount of ADA.
+  - DRep registration form - UI component allowing to register as a DRep. Under the hood metadata anchor is mocked with provided default values.
+    DRep uses following CSL services:
+  - TransactionBuilder - to build the transaction
+  - CertificatesBuilder - to build the DRep registration certificate
+  - DrepRegistration - to build the DRep registration certificate
+  - DrepDeregistration - to build the DRep deregistration certificate
+  - DrepUpdate - to build the DRep update certificate
+
+**Note**
+
+- Sole Voter can become a DRep by providing the metadata.
+- DRep can become a Sole Voter by delegating the ADA to themselves.
+
+- **DRep directory** - DRep directory is a list of all registered DReps. It displays DRep metadata, and amount of Voting Power. DRep directory is visible for all users. DRep directory is the part of delegation pillar.
+
+  DRep directory allows to delegate ADA to DReps. It uses following CSL services:
+
+  - Credential
+  - DRep
+  - Certificate
+  - VoteDelegation
+
+- **GA Submission** - GA Submission is a form allowing to submit a Governance Action. GA Submission is the part of governance pillar. GA Submission uses following CSL services:
+
+  - TransactionBuilder - to build the transaction
+  - CertificatesBuilder - to build the governance action certificate
+  - GovernanceAction - to build the governance action certificate
+
+  Additionaly, GA Submission uses Metadata validation service to validate the metadata.
+
+## Backend
+
+# Technology Stack
+
+- [Haskell](https://www.haskell.org/): An advanced, purely functional programming language.
+- [cardano-node](https://github.com/IntersectMBO/cardano-node): The core component that is used to participate in a Cardano decentralised blockchain.
+- [cardano-db-sync](https://github.com/IntersectMBO/cardano-db-sync): A component that follows the Cardano chain and stores blocks and transactions in PostgreSQL.
+
+# Description
+
+The backend of our application is built using Haskell, a purely functional programming language known for its strong static typing, high-level abstractions, and expressive syntax.
+
+Our backend interacts with the Cardano blockchain through two main components: `cardano-node` and `cardano-db-sync`.
+
+# Components
+
+- **Cardano Node**
+
+The `cardano-node` is the core component that our backend uses to participate in the Cardano decentralized blockchain. It allows our application to create transactions, submit them to the network, and listen for new blocks and transactions.
+
+- **Cardano DB-Sync**
+
+The `cardano-db-sync` component follows the Cardano chain and stores blocks and transactions in a PostgreSQL database. This allows our application to query blockchain data in a structured and efficient manner.
+
+## Application Logic
+
+The application logic is implemented in Haskell. It handles requests from the frontend, reads from the DB-Sync, and sends responses back to the frontend.
+
+## Data Storage
+
+Our application uses PostgreSQL for data storage. The `cardano-db-sync` component populates the database with blockchain data, and our application logic also stores additional data as needed, such as governance actions.
+
+## Architecture diagram
+
+**Valid as of: 2024-06-06**
+![Architecture diagram](<Architecture diagram.png>)

govtool/backend/app/Main.hs

@@ -8,6 +8,7 @@
 
 module Main where
 
+import           Control.Concurrent.QSem                (newQSem)
 import           Control.Exception                      (Exception,
                                                          SomeException,
                                                          fromException, throw)
@@ -115,7 +116,8 @@ startApp vvaConfig = do
     dRepVotingPowerCache <- newCache
     dRepListCache <- newCache
     networkMetricsCache <- newCache
-    metadataValidationCache <- newCache
+    proposalMetadataValidationCache <- newCache
+    dRepMetadataValidationCache <- newCache
     return $ CacheEnv
       { proposalListCache
       , getProposalCache
@@ -127,12 +129,13 @@ startApp vvaConfig = do
       , dRepVotingPowerCache
       , dRepListCache
       , networkMetricsCache
-      , metadataValidationCache
+      , proposalMetadataValidationCache
+      , dRepMetadataValidationCache
       }
   connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 1 60
   vvaTlsManager <- newManager tlsManagerSettings
-
-  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager}
+  qsem <- newQSem (metadataValidationMaxConcurrentRequests vvaConfig)
+  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager, vvaMetadataQSem=qsem}
   server' <- mkVVAServer appEnv
   runSettings settings server'
 

govtool/backend/example-config.json

@@ -11,5 +11,6 @@
     "cachedurationseconds": 20,
     "sentrydsn": "https://username:[email protected]/id",
     "metadatavalidationhost": "localhost",
-    "metadatavalidationport": 3001
+    "metadatavalidationport": 3001,
+    "metadatavalidationmaxconcurrentrequests": 10
 }