RPGF Attestations discovery (#329)

* add alphabetic index

* paginate applications

* rename applications to projects on graphql

* remove console logs

* fix id

* filter projects by category

* add projects search on display name

* include description words to search tri

* refine search

* add application aggregates

* fix aggregate loader

* add shuffle sort

* organize

* fix imports

* add sort and aggregate indexes for lists

* apply pagination, sorts, filters & search to lists

* pull by id & aggregates

* resolve projects in list content

* add isCitizen flag to delegates

* add isCitizen flag to graphql

* remove test for za string encoding

* sort lists by likes

* all likes to list schema

* add likedBy filter to lists

* add byIncludedInBallots sort on projects

* update dev DO instance name

* fix search trie for full words

* merge entities for indexers

* add filterFn to search

* revert entityDefinitions merge

* fix search & filter combo

* fix aggregates resolver

* set databalse url in graphql request if not set

packages/backend/codegen.yaml

@@ -53,5 +53,5 @@ generates:
         DelegateStatement: ../delegateStatement#DelegateStatementModel
 
         RetroPGF: ../retroPGF#RetroPGFModel
-        Application: ../retroPGF#ApplicationModel
+        Project: ../retroPGF#ProjectModel
         List: ../retroPGF#ListModel

packages/backend/src/bin/server.ts

@@ -22,6 +22,8 @@ import { EthersLogProvider } from "../indexer/logProvider/logProvider";
 import { makeLatestBlockFetcher } from "../schema/latestBlockFetcher";
 import { startRestServer } from "./restServer";
 import { startProxyServer } from "./proxyServer";
+import { makeLikesStore } from "../services/likes";
+import { makeBallotsStore } from "../services/ballot";
 
 async function main() {
   const schema = makeGatewaySchema();
@@ -91,6 +93,8 @@ async function main() {
           },
         },
         tracingContext: makeEmptyTracingContext(),
+        likesStore: makeLikesStore(),
+        ballotsStore: makeBallotsStore(),
       };
     },
     port: 4002,

packages/backend/src/indexer/bin/backfill.ts

@@ -6,7 +6,7 @@ import {
   BlockIdentifier,
   makeStorageHandleWithStagingArea,
 } from "../storageHandle";
-import { indexers } from "../contracts";
+import { entityDefinitions, indexers } from "../contracts";
 import { StructuredError } from "../utils/errorUtils";
 import { ethers } from "ethers";
 import { LevelEntityStore } from "../storage/level/levelEntityStore";
@@ -49,8 +49,6 @@ async function main() {
     return;
   }
 
-  const entityDefinitions = combineEntities(indexers);
-
   const progressBar = makeProgressBar(highestCommonBlock.blockNumber);
 
   let idx = 0;
@@ -88,7 +86,7 @@ async function main() {
       const [storageHandle, loadedEntities] = makeStorageHandleWithStagingArea(
         entityBlockStagingArea,
         store,
-        indexer.entities
+        entityDefinitions
       );
 
       try {

packages/backend/src/indexer/contracts/EAS.ts

@@ -1,13 +1,34 @@
 import { ethers, BigNumber } from "ethers";
+import { entityDefinitions } from ".";
 import { makeContractInstance } from "../../contracts";
 import { EAS__factory } from "../../contracts/generated";
 import { makeEntityDefinition, makeIndexerDefinition } from "../process";
 import * as serde from "../serde";
+import {
+  efficientLengthEncodingNaturalPositiveNumbers,
+  efficientLengthEncodingStringAsc,
+  efficientLengthEncodingStringDesc,
+} from "../utils/efficientLengthEncoding";
+import {
+  governanceTokenIndexer,
+  loadAccount,
+  saveAccount,
+} from "./GovernanceToken";
+import {
+  attestationAggregatesEntityDefinitions,
+  updateApplicationsAggregate,
+  updateListsAggregate,
+} from "./utils/aggregates";
+import {
+  addToApplicationsTrie,
+  addToListsTrie,
+  serchEntityDefinitions,
+} from "./utils/search";
 
 const EASContract = makeContractInstance({
   iface: EAS__factory.createInterface(),
   address: "0x4200000000000000000000000000000000000021",
-  startingBlock: 6490467,
+  startingBlock: 107953260,
 });
 
 const rpgfSchemas: { [key: string]: string } = {
@@ -24,6 +45,10 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
   name: "EAS",
 
   entities: {
+    ...serchEntityDefinitions,
+    ...attestationAggregatesEntityDefinitions,
+    ...governanceTokenIndexer.entities,
+
     Badgeholder: makeEntityDefinition({
       serde: serde.object({
         uid: serde.string,
@@ -91,24 +116,27 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
         certifiedNotDesignatedOrSanctionedOrBlocked: serde.boolean,
         certifiedNotSponsoredByPoliticalFigureOrGovernmentEntity: serde.boolean,
         certifiedNotBarredFromParticipating: serde.boolean,
+
+        blockNumber: serde.bigNumber,
       }),
       indexes: [
         {
-          indexName: "byDisplayName",
+          indexName: "byNameAZ",
           indexKey({ displayName }) {
-            return displayName;
+            return efficientLengthEncodingStringDesc(displayName);
           },
         },
         {
-          indexName: "byApplicantType",
-          indexKey({ applicantType }) {
-            return applicantType;
+          indexName: "byNameZA",
+          indexKey({ displayName }) {
+            return efficientLengthEncodingStringAsc(displayName);
           },
         },
         {
-          indexName: "byImpactCategory",
-          indexKey({ impactCategory }) {
-            return impactCategory[0]; // TODO: support multiple categories
+          // This index is used for shuffle
+          indexName: "byBlockNumber",
+          indexKey({ blockNumber }) {
+            return efficientLengthEncodingNaturalPositiveNumbers(blockNumber);
           },
         },
       ],
@@ -132,12 +160,29 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
             OPAmount: serde.number,
           })
         ),
+
+        categories: serde.array(serde.string),
+
+        blockNumber: serde.bigNumber,
       }),
       indexes: [
         {
-          indexName: "byListName",
+          indexName: "byNameAZ",
           indexKey({ listName }) {
-            return listName;
+            return efficientLengthEncodingStringDesc(listName);
+          },
+        },
+        {
+          indexName: "byNameZA",
+          indexKey({ listName }) {
+            return efficientLengthEncodingStringAsc(listName);
+          },
+        },
+        {
+          // This index is used for shuffle
+          indexName: "byBlockNumber",
+          indexKey({ blockNumber }) {
+            return efficientLengthEncodingNaturalPositiveNumbers(blockNumber);
           },
         },
       ],
@@ -153,7 +198,7 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
       async handle(
         handle,
         { args: { recipient, attester, uid, schema } },
-        _,
+        { blockNumber },
         { easDataFetcher, asyncDataFetcher }
       ) {
         const schemaName = rpgfSchemas[schema];
@@ -165,8 +210,9 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
             case "BADGEHOLDERS": {
               const badgeholderSchema = decodeBadgeholderSchema(schemaData);
 
+              // TODO: check if correct round
               if (attester == badgeholderAttester) {
-                handle.saveEntity("Badgeholder", uid, {
+                await handle.saveEntity("Badgeholder", uid, {
                   uid,
                   recipient,
                   attester,
@@ -177,6 +223,11 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
                   referredBy: badgeholderSchema.referredBy,
                   referredMethod: badgeholderSchema.referredMethod,
                 });
+
+                console.log("saving badgeholder", uid, recipient, attester);
+                const delegate = await loadAccount(handle, recipient);
+                console.log("delegate", delegate);
+                saveAccount(handle, { ...delegate, isCitizen: true });
               }
 
               break;
@@ -193,7 +244,7 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
                 );
               }
 
-              handle.saveEntity("Application", uid, {
+              const application = {
                 uid,
                 recipient,
                 attester,
@@ -226,7 +277,12 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
                   applicaitonData.certifiedNotSponsoredByPoliticalFigureOrGovernmentEntity,
                 certifiedNotBarredFromParticipating:
                   applicaitonData.certifiedNotBarredFromParticipating,
-              });
+                blockNumber: BigNumber.from(blockNumber),
+              };
+
+              handle.saveEntity("Application", uid, application);
+              await updateApplicationsAggregate(handle, application);
+              await addToApplicationsTrie(handle, application);
 
               break;
             }
@@ -242,7 +298,20 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
                 );
               }
 
-              handle.saveEntity("List", uid, {
+              const categories = new Set<string>();
+              for await (const vote of listData.listContent) {
+                const application = await handle.loadEntity(
+                  "Application",
+                  vote.RPGF3_Application_UID
+                );
+                if (application) {
+                  application.impactCategory.forEach((category) => {
+                    categories.add(category);
+                  });
+                }
+              }
+
+              const list = {
                 uid,
                 recipient,
                 attester,
@@ -255,7 +324,15 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
                   listData.impactEvaluationDescription,
                 impactEvaluationLink: listData.impactEvaluationLink,
                 listContent: listData.listContent,
-              });
+
+                categories: [...categories],
+
+                blockNumber: BigNumber.from(blockNumber),
+              };
+
+              handle.saveEntity("List", uid, list);
+              await updateListsAggregate(handle, list);
+              await addToListsTrie(handle, list);
 
               break;
             }
@@ -279,6 +356,8 @@ export const EASIndexer = makeIndexerDefinition(EASContract, {
             ...badgeholder,
             revokedAtBlock: BigNumber.from(log.blockNumber),
           });
+          const delegate = await loadAccount(handle, badgeholder.recipient);
+          saveAccount(handle, { ...delegate, isCitizen: false });
         }
 
         if (application) {

packages/backend/src/indexer/contracts/GovernanceToken.ts

@@ -9,7 +9,7 @@ import { GovernanceToken__factory } from "../../contracts/generated";
 import { BigNumber, ethers } from "ethers";
 import * as serde from "../serde";
 import { RuntimeType } from "../serde";
-import { encodeOrdinal, logToOrdinal, ordinal } from "./ordinal";
+import { encodeOrdinal, logToOrdinal, ordinal } from "./utils/ordinal";
 
 const governanceTokenContract = makeContractInstance({
   iface: GovernanceToken__factory.createInterface(),
@@ -53,6 +53,8 @@ export const governanceTokenIndexer = makeIndexerDefinition(
           tokensRepresented: serde.bigNumber,
           delegatingTo: serde.string,
           accountsRepresentedCount: serde.bigNumber,
+
+          isCitizen: serde.boolean,
         }),
         indexes: [
           {
@@ -207,10 +209,11 @@ export function defaultAccount(
     tokensRepresented: ethers.BigNumber.from(0),
     accountsRepresentedCount: BigNumber.from(1),
     delegatingTo: ethers.constants.AddressZero,
+    isCitizen: false,
   };
 }
 
-async function loadAccount(
+export async function loadAccount(
   // @ts-ignore
   handle: StorageHandleForIndexer<typeof governanceTokenIndexer>,
   from: string
@@ -220,7 +223,7 @@ async function loadAccount(
   return (await handle.loadEntity("Address", from)) ?? defaultAccount(from);
 }
 
-function saveAccount(
+export function saveAccount(
   // @ts-ignore
   handle: StorageHandleForIndexer<typeof governanceTokenIndexer>,
   entity: RuntimeType<

packages/backend/src/indexer/contracts/OptimismGovernor.ts

@@ -10,7 +10,7 @@ import * as serde from "../serde";
 import { RuntimeType } from "../serde";
 import { efficientLengthEncodingNaturalNumbers } from "../utils/efficientLengthEncoding";
 import { makeCompoundKey } from "../indexKey";
-import { encodeOrdinal, logToOrdinal, ordinal } from "./ordinal";
+import { encodeOrdinal, logToOrdinal, ordinal } from "./utils/ordinal";
 
 export const governorContract = makeContractInstance({
   iface: OptimismGovernorV5__factory.createInterface(),

packages/backend/src/indexer/contracts/OptimismGovernorTest.ts

@@ -10,7 +10,7 @@ import * as serde from "../serde";
 import { RuntimeType } from "../serde";
 import { efficientLengthEncodingNaturalNumbers } from "../utils/efficientLengthEncoding";
 import { makeCompoundKey } from "../indexKey";
-import { encodeOrdinal, logToOrdinal, ordinal } from "./ordinal";
+import { encodeOrdinal, logToOrdinal, ordinal } from "./utils/ordinal";
 
 export const governorContractTest = makeContractInstance({
   iface: OptimismGovernorV5__factory.createInterface(),