Fix#400: Re-merge YAML & CLI

.github/workflows/check_mainnet.yml

@@ -17,18 +17,20 @@ jobs:
         with:
           python-version: "3.11"
 
+      - name: Fetch the address file and move it to contracts directory
+        run: |
+          wget https://raw.githubusercontent.com/oceanprotocol/contracts/main/addresses/address.json
+          mkdir -p ~/.ocean/ocean-contracts/artifacts/
+          mv address.json ~/.ocean/ocean-contracts/artifacts/
+
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
       - name: Notify Slack
-        env:
-          RPC_URL: "https://sapphire.oasis.io"
-          SUBGRAPH_URL: "https://v4.subgraph.sapphire-mainnet.oceanprotocol.com/subgraphs/name/oceanprotocol/ocean-subgraph"
-          PRIVATE_KEY: "0xb23c44b8118eb7a7f70d21b0d20aed9b05d85d22ac6a0e57697c564da1c35554"
         run: |
-          output=$(python scripts/check_network.py 1 | grep -E 'FAIL|WARNING|error' || true)
+          output=$(python pdr check_network ppss.yaml sapphire-mainnet | grep -E 'FAIL|WARNING|error' || true)
           fact=$(curl -s https://catfact.ninja/fact | jq -r '.fact')
           if [ -z "$output" ]; then
             echo "No output, so no message will be sent to Slack"

.github/workflows/check_testnet.yml

@@ -17,18 +17,20 @@ jobs:
         with:
           python-version: "3.11"
 
+      - name: Fetch the address file and move it to contracts directory
+        run: |
+          wget https://raw.githubusercontent.com/oceanprotocol/contracts/main/addresses/address.json
+          mkdir -p ~/.ocean/ocean-contracts/artifacts/
+          mv address.json ~/.ocean/ocean-contracts/artifacts/
+
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
       - name: Notify Slack
-        env:
-          RPC_URL: "https://testnet.sapphire.oasis.dev"
-          SUBGRAPH_URL: "https://v4.subgraph.sapphire-testnet.oceanprotocol.com/subgraphs/name/oceanprotocol/ocean-subgraph"
-          PRIVATE_KEY: "0xb23c44b8118eb7a7f70d21b0d20aed9b05d85d22ac6a0e57697c564da1c35554"
         run: |
-          output=$(python scripts/check_network.py 1 | grep -E 'FAIL|WARNING|error' | grep -v "1h" || true)
+          output=$(python pdr check_network ppss.yaml sapphire-testnet | grep -E 'FAIL|WARNING|error' | grep -v "1h" || true)
           joke=$(curl -s https://official-joke-api.appspot.com/jokes/general/random | jq -r '.[0].setup, .[0].punchline')
           if [ -z "$output" ]; then
             echo "No output, so no message will be sent to Slack"

.github/workflows/cron_topup.yml

@@ -3,7 +3,7 @@ name: Topup accounts
 on:
   schedule:
     - cron: "0 * * * *"
-  
+
 jobs:
   topup-mainnet:
     runs-on: ubuntu-latest
@@ -15,18 +15,20 @@ jobs:
           uses: actions/setup-python@v2
           with:
             python-version: "3.11"
-
+        - name: Fetch the address file and move it to contracts directory
+          run: |
+            wget https://raw.githubusercontent.com/oceanprotocol/contracts/main/addresses/address.json
+            mkdir -p ~/.ocean/ocean-contracts/artifacts/
+            mv address.json ~/.ocean/ocean-contracts/artifacts/
         - name: Install Dependencies
           run: |
             python -m pip install --upgrade pip
             pip install -r requirements.txt
         - name: Set env variables
           run: |
-            echo "SUBGRAPH_URL=http://v4.subgraph.sapphire-mainnet.oceanprotocol.com/subgraphs/name/oceanprotocol/ocean-subgraph" >> $GITHUB_ENV
-            echo "RPC_URL=https://sapphire.oasis.io" >> $GITHUB_ENV
             echo "PRIVATE_KEY=${{ secrets.TOPUP_SCRIPT_PK }}" >> $GITHUB_ENV
         - name: Run top-up script
-          run: python3 scripts/topup.py
+          run: python3 pdr topup ppss.yaml sapphire-mainnet
 
   topup-testnet:
     runs-on: ubuntu-latest
@@ -41,10 +43,13 @@ jobs:
           run: |
             python -m pip install --upgrade pip
             pip install -r requirements.txt
+        - name: Fetch the address file and move it to contracts directory
+          run: |
+            wget https://raw.githubusercontent.com/oceanprotocol/contracts/main/addresses/address.json
+            mkdir -p ~/.ocean/ocean-contracts/artifacts/
+            mv address.json ~/.ocean/ocean-contracts/artifacts/
         - name: Set env variables
           run: |
-            echo "SUBGRAPH_URL=http://v4.subgraph.sapphire-testnet.oceanprotocol.com/subgraphs/name/oceanprotocol/ocean-subgraph" >> $GITHUB_ENV
-            echo "RPC_URL=https://testnet.sapphire.oasis.dev" >> $GITHUB_ENV
             echo "PRIVATE_KEY=${{ secrets.TOPUP_SCRIPT_PK }}" >> $GITHUB_ENV
         - name: Run top-up script
-          run: python3 scripts/topup.py
+          run: python3 pdr topup ppss.yaml sapphire-testnet

.github/workflows/pytest.yml

@@ -29,7 +29,6 @@ jobs:
         name: Checkout Barge
         with:
           repository: "oceanprotocol/barge"
-          ref: "main"
           path: "barge"
 
       - name: Run Barge
@@ -57,5 +56,10 @@ jobs:
       - name: Test with pytest
         id: pytest
         run: |
-          coverage run --omit="*test*" -m pytest
+          coverage run --source=pdr_backend --omit=*/test/*,*/test_ganache/*,*/test_noganache/* -m pytest
           coverage report
+          coverage xml
+      - name: Publish code coverage
+        uses: paambaati/[email protected]
+        env:
+          CC_TEST_REPORTER_ID: ${{secrets.CC_TEST_REPORTER_ID}}

.gitignore

@@ -164,9 +164,12 @@ cython_debug/
 .test_cache/
 .cache/
 
-# predictoor dynamic modeling
+# predictoor-specific
 out*.txt
+my_ppss.yaml
 csvs/
+parquet_data/
+
 # pdr_backend accuracy output
 pdr_backend/accuracy/output/*.json
 # pm2 configs

.pylintrc

@@ -119,7 +119,8 @@ disable=too-many-locals,
 	consider-using-dict-items,
         consider-using-generator,
 	dangerous-default-value,
-	unidiomatic-typecheck
+	unidiomatic-typecheck,
+	unsubscriptable-object
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

README.md

@@ -5,44 +5,79 @@ SPDX-License-Identifier: Apache-2.0
 
 # pdr-backend
 
+⚠️ As of v0.2, the CLI replaces previous `main.py` calls. Update your flows accordingly.
+
 ## Run bots (agents)
 
 - **[Run predictoor bot](READMEs/predictoor.md)** - make predictions, make $
 - **[Run trader bot](READMEs/trader.md)** - consume predictions, trade, make $
 
+
 (If you're a predictoor or trader, you can safely ignore the rest of this README.)
 
+## Settings: PPSS
+
+A "ppss" yaml file, like [`ppss.yaml`](ppss.yaml), holds parameters for all bots and simulation flows.
+- We follow the idiom "pp" = problem setup (what to solve), "ss" = solution strategy (how to solve).
+- `PRIVATE_KEY` is an exception; it's set as an envvar.
+
+When you run a bot from the CLI, you specify your PPSS YAML file.
+
+## CLI
+
+(First, [install pdr-backend](READMEs/predictoor.md#install-pdr-backend-repo) first.)
+
+To see CLI options, in console:
+```console
+pdr
+```
+
+This will output something like:
+```text
+Usage: pdr sim|predictoor|trader|..
+
+Main tools:
+  pdr sim YAML_FILE
+  pdr predictoor APPROACH YAML_FILE NETWORK
+  pdr trader APPROACH YAML_FILE NETWORK
+...
+```
 
 ## Atomic READMEs
 
 - [Get tokens](READMEs/get-tokens.md): [testnet faucet](READMEs/testnet-faucet.md), [mainnet ROSE](READMEs/get-rose-on-sapphire.md) & [OCEAN](READMEs/get-ocean-on-sapphire.md)
-- [Envvars](READMEs/envvars.md)
-- [Predictoor subgraph](READMEs/subgraph.md)
-- [Dynamic model codebase](READMEs/dynamic-model-codebase.md)
-- [Static models in predictoors](READMEs/static-model.md)
+- [Claim payout for predictoor bot](READMEs/payout.md)
+- [Predictoor subgraph](READMEs/subgraph.md). [Subgraph filters](READMEs/filters.md)
+- [Run barge locally](READMEs/barge.md)
 
 ## Flows for core team
 
-- **Backend dev** - for `pdr-backend` itself
-  - [Main backend-dev README](READMEs/backend-dev.md)
+- Backend-dev - for `pdr-backend` itself
+  - [Local dev flow](READMEs/dev.md)
+  - [VPS dev flow](READMEs/vps.md)
   - [Release process](READMEs/release-process.md)
-  - [Run barge locally](READMEs/barge.md)
-  - [Run barge remotely on VPS](READMEs/vps.md)
-  - [MacOS gotchas](READMEs/macos.md) wrt Docker & ports
-- **[Run dfbuyer bot](READMEs/dfbuyer.md)** - runs Predictoor DF rewards
-- **[Run publisher](READMEs/publisher.md)** - publish new feeds
-- **[Scripts](scripts/)** for performance stats, more
+  - [Clean code guidelines](READMEs/clean-code.md)
+- [Run dfbuyer bot](READMEs/dfbuyer.md) - runs Predictoor DF rewards
+- [Run publisher](READMEs/publisher.md) - publish new feeds
+- [Run trueval](READMEs/trueval.md) - run trueval bot
 
 ## Repo structure
 
 This repo implements all bots in Predictoor ecosystem.
 
-Each bot has a directory:
-- `predictoor` - submits individual predictions
-- `trader` - buys aggregated predictions, then trades
-- other bots: `trueval` report true values to contract, `dfbuyer` implement Predictoor Data Farming, `publisher` to publish
+Each bot has a directory. Alphabetically:
+- `dfbuyer` - buy feeds on behalf of Predictoor DF
+- `predictoor` - submit individual predictions
+- `publisher` - publish pdr data feeds
+- `trader` - buy aggregated predictions, then trade
+- `trueval` - report true values to contract
 
-Other directories:
-- `util` - tools for use by any agent
-- `models` - classes that wrap Predictoor contracts; for setup (BaseConfig); and for data feeds (Feed)
+Other directories, alphabetically:
+- `accuracy` - calculates % correct, for display in predictoor.ai webapp
+- `data_eng` - data engineering & modeling
+- `models` - class-based data structures, and classes to wrap contracts
+- `payout` - OCEAN & ROSE payout
+- `ppss` - settings
+- `sim` - simulation flow
+- `util` - function-based tools
 

READMEs/barge-calls.md

@@ -0,0 +1,41 @@
+### Barge flow of calls
+
+From getting barge going, here's how it calls specific pdr-backend components and passes arguments.
+
+- user calls `/barge/start_ocean.sh` to get barge going
+  - then, `start_ocean.sh` fills `COMPOSE_FILES` incrementally. Eg `COMPOSE_FILES+=" -f ${COMPOSE_DIR}/pdr-publisher.yml"`
+     - `barge/compose-files/pdr-publisher.yml` sets:
+       - `pdr-publisher: image: oceanprotocol/pdr-backend:${PDR_BACKEND_VERSION:-latest}`
+       - `pdr-publisher: command: publisher`
+       - `pdr-publisher: networks: backend: ipv4_address: 172.15.0.43`
+       - `pdr-publisher: environment:`
+         - `RPC_URL: ${NETWORK_RPC_URL}` (= `http://localhost:8545` via `start_ocean.sh`)
+         - `ADDRESS_FILE: /root/.ocean/ocean-contracts/artifacts/address.json`
+	 - (many `PRIVATE_KEY_*`)
+
+  - then, `start_ocean.sh` pulls the `$COMPOSE_FILES` as needed:
+    - `[ ${FORCEPULL} = "true" ] && eval docker-compose "$DOCKER_COMPOSE_EXTRA_OPTS" --project-name=$PROJECT_NAME "$COMPOSE_FILES" pull`
+
+  - then, `start_ocean.sh` runs docker-compose including all `$COMPOSE_FILES`:
+    - `eval docker-compose "$DOCKER_COMPOSE_EXTRA_OPTS" --project-name=$PROJECT_NAME "$COMPOSE_FILES" up --remove-orphans`
+    - it executes each of the `"command"` entries in compose files.
+       - (Eg for pdr-publisher.yml, `"command" = "publisher ppss.yaml development"`)
+       - Which then goes to `pdr-backend/entrypoint.sh` via `"python /app/pdr_backend/pdr $@"`
+         - (where `@` is unpacked as eg `publisher ppss.yaml development`) [Ref](https://superuser.com/questions/1586997/what-does-symbol-mean-in-the-context-of#:).
+	 - Then it goes through the usual CLI at `pdr-backend/pdr_backend/util/cli_module.py`
+
+
+### How to make changes to calls
+
+If you made a change to pdr-backend CLI interface, then barge must call using the updated CLI command.
+
+How:
+- change the relevant compose file's `"command"`. Eg change `barge/compose-files/pdr-publisher.yml`'s "command" value to `publisher ppss.yaml development`
+- also, change envvar setup as needed. Eg in compose file, remove `RPC_URL` and `ADDRESS_FILE` entry.
+- ultimately, ask: "does Docker have everything it needs to succesfully run the component?"
+
+### All Barge READMEs
+
+- [barge.md](barge.md): the main Barge README
+- [barge-calls.md](barge-calls.md): order of execution from Barge and pdr-backend code
+- [release-process.md](release-process.md): pdr-backend Dockerhub images get published with each push to `main`, and sometimes other branches. In turn these are used by Barge.