From 0b1bd5d9befdf4a91ec49b8da2bb9fb137024b3c Mon Sep 17 00:00:00 2001 From: Vishal <64505169+vishalxl@users.noreply.github.com> Date: Sat, 24 Dec 2022 01:35:33 +0530 Subject: [PATCH] getting only 200 maximum contacts for any user reduced calls made. combined user fetch and mention fetch to one request. --- bin/nostr_console.dart | 56 +++++++++++++++++++---------------------- lib/relays.dart | 17 ++++++++++++- lib/settings.dart | 4 +-- lib/utils.dart | 20 ++++++++++++--- pubspec.yaml | 1 + scripts/configfile.cfg | 19 ++++++++++++++ scripts/test_servers.sh | 2 +- 7 files changed, 82 insertions(+), 37 deletions(-) diff --git a/bin/nostr_console.dart b/bin/nostr_console.dart index db26b0f..d2869d2 100644 --- a/bin/nostr_console.dart +++ b/bin/nostr_console.dart @@ -259,7 +259,6 @@ Future main(List arguments) async { // if more than 1000 posts have already been read from the file, then don't get too many day's events. Only for last 3 days. if(numFileEvents > 1000) { - limitPerSubscription = 5000; limitSelfEvents = 10; limitOthersEvents = 3; gDefaultNumWaitSeconds = gDefaultNumWaitSeconds ~/5; @@ -280,8 +279,7 @@ Future main(List arguments) async { } if( userPublicKey!= "") { - getUserEvents(gListRelayUrls1, userPublicKey, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents)); - getMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); + getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); } Future.delayed(Duration(milliseconds: numWaitSeconds), () { @@ -310,29 +308,19 @@ Future main(List arguments) async { // get event for user if( userPublicKey!= "") { - getUserEvents(gListRelayUrls1, userPublicKey, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents)); - getMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); + getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); } - //getKindEvents([gSecretMessageKind], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo( limitSelfEvents)); - - Set usersFetched = {userPublicKey}; - // remove user from default list if he exists in it. because theyv'e already been fetched. - gDefaultFollows = gDefaultFollows.difference(usersFetched); - - // get other user events - getMultiUserEvents(gListRelayUrls1, gDefaultFollows, 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); - usersFetched = usersFetched.union(gDefaultFollows); // get group and meta info events getKindEvents([40, 41], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents)); - getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(4)); - - getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), {0,3}); + getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); + + // get default users; remove user from default list if user exists in it. because theyv'e already been fetched. + getMultiUserEvents(gListRelayUrls1, gDefaultFollows.difference({userPublicKey}), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); + Set usersFetched = gDefaultFollows.union({userPublicKey}); stdout.write('Waiting for user posts to come in.....'); Future.delayed( Duration(milliseconds: gDefaultNumWaitSeconds), () { - //print("total users fetched: ${usersFetched.length}"); - initialEvents.addAll(getRecievedEvents()); clearEvents(); @@ -343,35 +331,43 @@ Future main(List arguments) async { initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event Set contacts = {}; + Set pTags = {}; + if( userPublicKey != "") { - // get the latest kind 3 event for the user, which lists his 'follows' list + // get the latest kind 3 event for the user, which has the 'follows' list Event? contactEvent = getContactEvent(userPublicKey); // if contact list was found, get user's feed; also get some default contacts if (contactEvent != null ) { if(gDebug > 0) print("In main: found contact list: \n ${contactEvent.originalJson}"); - contactEvent.eventData.contactList.forEach((contact) { + contactEvent.eventData.contactList.forEach((contact) { contacts.add(contact.id); }); - contacts = contacts.difference(usersFetched); // remove already fetched users from this list - - getContactFeed(gListRelayUrls1, contacts, 3 * gLimitPerSubscription, getSecondsDaysAgo( limitOthersEvents)); - usersFetched = usersFetched.union(contacts); } else { print("Could not find your contact list."); } } // fetch extra events for people who don't have too large a follow list - if( usersFetched.length < gMaxPtagsToGet * 2 ) { + if( contacts.union(gDefaultFollows).length < gMaxPtagsToGet ) { // calculate top mentioned ptags, and then get the events for those users - Set pTags = getpTags(initialEvents, gMaxPtagsToGet); - pTags = pTags.difference(usersFetched); + pTags = getpTags(initialEvents, gMaxPtagsToGet); + } - getMultiUserEvents(gListRelayUrls1, pTags, 4 * gLimitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); - usersFetched = usersFetched.union(pTags); + // get only 200 contacts maximum + int maxContactsFetched = 200; + if( contacts.length > maxContactsFetched) { + int i = 0; + contacts.retainWhere((element) => i++ > 200); // retain only first 200, whichever they may be } + getMultiUserEvents(gListRelayUrls1, contacts.union(pTags).difference(usersFetched), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); + usersFetched = usersFetched.union(contacts).union(pTags); + + // get meta events of all users fetched + print("getting meta for # users : ${usersFetched.length} #contacts = ${contacts.length}"); + getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents*2), {0,3}); + // get events from channels of user Set userChannels = getUserChannels(initialEvents, userPublicKey); //getMentionEvents(gListRelayUrls1, userChannels, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#e"); diff --git a/lib/relays.dart b/lib/relays.dart index 6f24b7d..7c4ccf9 100644 --- a/lib/relays.dart +++ b/lib/relays.dart @@ -102,6 +102,14 @@ class Relays { sendRequest(relayUrl, request); } + void getIdAndMentionEvents(String relayUrl, Set ids, int limit, int sinceWhen, String tagToGet) { + + String subscriptionId = "id_mention_${tagToGet}" + (relays[relayUrl]?.numRequestsSent??"").toString() + "_" + relayUrl.substring(6); + String request = getIdAndMentionRequest(subscriptionId, ids, limit, sinceWhen, tagToGet); + sendRequest(relayUrl, request); + } + + /* * @connect Connect to given relay and get all events for multiple users/publicKey and insert the * received events in the given List @@ -255,6 +263,13 @@ void getMentionEvents(Set serverUrls, Set ids, int numUserEvents }); } +void getIdAndMentionEvents(Set serverUrls, Set ids, int numUserEvents, int sinceWhen, String tagToGet) { + serverUrls.forEach((serverUrl) { + relays.getIdAndMentionEvents(serverUrl, ids, numUserEvents, sinceWhen, tagToGet); + }); +} + + getKindEvents(List kind, Set serverUrls, int limit, int sinceWhen) { serverUrls.forEach((serverUrl) { relays.getKindEvents(kind, serverUrl, limit, sinceWhen); @@ -283,7 +298,7 @@ void sendEventsRequest(Set serverUrls, Set eventIds) { if( eventIds.length == 0) return; - String eventIdsStr = getJsonList(eventIds);; + String eventIdsStr = getCommaSeparatedQuotedStrs(eventIds);; String getEventRequest = '["REQ","event_${eventIds.length}",{"ids":[$eventIdsStr]}]'; if( gDebug > 0) log.info("sending $getEventRequest"); diff --git a/lib/settings.dart b/lib/settings.dart index e2b55d8..9210ba3 100644 --- a/lib/settings.dart +++ b/lib/settings.dart @@ -34,13 +34,13 @@ bool gOverWriteFile = false; // overwrite the file, and don't ju const int gDontAddToStoreBeforeDays = 60; // events older than this are not added to the Store of all events const int gLimitFollowPosts = 20; // when getting events, this is the since field (unless a fully formed request is given in command line) -const int gLimitPerSubscription = 10000; +const int gLimitPerSubscription = 20000; // don't show notifications for events that are older than 5 days and come when program is running // applicable only for notifications and not for search results. Search results set a flag in EventData and don't use this variable const int gDontHighlightEventsOlderThan = 4; -int gDefaultNumWaitSeconds = 12000; // is used in main() +int gDefaultNumWaitSeconds = 10000; // is used in main() const int gMaxAuthorsInOneRequest = 300; // number of author requests to send in one request const int gMaxPtagsToGet = 100; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent) diff --git a/lib/utils.dart b/lib/utils.dart index 81f306c..aa13e6b 100644 --- a/lib/utils.dart +++ b/lib/utils.dart @@ -364,7 +364,7 @@ String getStringFromUser(String prompt, [String defaultValue=""] ) { // returns list in form ( if 3 sized list) // "pubkey1","pubkey2","pubkey3" -String getJsonList(Set publicKeys) { +String getCommaSeparatedQuotedStrs(Set publicKeys) { String s = ""; int i = 0; for(String pubkey in publicKeys) { @@ -448,9 +448,23 @@ String getMentionRequest(String subscriptionId, Set ids, int numUserEven } var strSubscription1 = '["REQ","$subscriptionId",{ "$tagToGet": ['; var strSubscription2 ='], "limit": $numUserEvents $strTime } ]'; - return strSubscription1 + getJsonList(ids) + strSubscription2; + return strSubscription1 + getCommaSeparatedQuotedStrs(ids) + strSubscription2; } +String getIdAndMentionRequest(String subscriptionId, Set ids, int numUserEvents, int sinceWhen, String tagToGet) { + String strTime = ""; + if( sinceWhen != 0) { + strTime = ', "since": ${sinceWhen.toString()}'; + } + + var strSubscription1 = '["REQ","$subscriptionId",{ "$tagToGet": ['; + var strSubscription2 ='], "limit": $numUserEvents $strTime } ]'; + String req = '["REQ","$subscriptionId",{ "$tagToGet": [' + getCommaSeparatedQuotedStrs(ids) + '], "limit": $numUserEvents $strTime},{"authors":[' + getCommaSeparatedQuotedStrs(ids) + ']} ]'; + //print("Created id and mention request: $req"); + return req; +} + + String getMultiUserRequest(String subscriptionId, Set publicKeys, int numUserEvents, int sinceWhen, [Set? kind = null]) { String strTime = ""; if( sinceWhen != 0) { @@ -467,7 +481,7 @@ String getMultiUserRequest(String subscriptionId, Set publicKeys, int nu var strSubscription1 = '["REQ","$subscriptionId",{ "authors": ['; var strSubscription2 ='],$strKindSection"limit": $numUserEvents $strTime } ]'; String s = ""; - s = getJsonList(publicKeys); + s = getCommaSeparatedQuotedStrs(publicKeys); String request = strSubscription1 + s + strSubscription2; //print("In getMultiUserRequest kind = $kind strKindSection = $strKindSection: request = $request"); return request; diff --git a/pubspec.yaml b/pubspec.yaml index 51b6a8f..d58cac9 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -13,6 +13,7 @@ homepage: https://github.com/vishalxl/nostr_console # reduced items fetched. 23/12 # reduced items more evening 23/12 +# reduced more evening 23/12 environment: sdk: '>=2.17.3 <3.0.0' diff --git a/scripts/configfile.cfg b/scripts/configfile.cfg index f12566e..ef75f6e 100644 --- a/scripts/configfile.cfg +++ b/scripts/configfile.cfg @@ -1,14 +1,33 @@ #nostr_servers=( "wss://relay.nostr.info" "wss://nostr-relay.wlvs.space" "wss://nostr-pub.wellorder.net" "wss://relay.damus.io" "wss://nostr.delo.software" "wss://nostr-pub.semisol.dev" "wss://nostr-relay-dev.wlvs.space") nostr_servers=( + "wss://nostr-relay-dev.wlvs.space" + "wss://nostr-01.bolt.observer" + "wss://nostr.shawnyeager.net" + "wss://nostr.zerofeerouting.com" + "wss://nostr.satsophone.tk" + "wss://relay.oldcity-bitcoiners.info" + "wss://relay.damus.io" "wss://relay.nostr.info" "wss://nostr-2.zebedee.cloud" + "wss://nostr.zebedee.cloud" "wss://nostr.semisol.dev" "wss://nostr.onsats.org" "wss://nostr-relay.wlvs.space" "wss://nostr-relay.digitalmob.ro" "wss://nostr.coinos.io" + + "wss://nostr-pub.wellorder.net" + "wss://nostr.radixrat.com" + "wss://nostr.bitcoiner.social" + + "wss://relay.valireum.net" + "wss://nostr-relay.trustbtc.org" + "wss://relay.stoner.com" + "wss://nostr.w3ird.tech" + "wss://nostr.bongbong.com" + "wss://nostr.hugo.md" ) diff --git a/scripts/test_servers.sh b/scripts/test_servers.sh index afe5f7d..abf1551 100644 --- a/scripts/test_servers.sh +++ b/scripts/test_servers.sh @@ -5,7 +5,7 @@ source ./configfile.cfg limit=100000 numHours=1 -echo -e "Requesting all evetns in last $numHours hours with a limit of $limit by executing the following command for each:" +echo -e "Requesting all events in last $numHours hours with a limit of $limit by executing the following command for each:" sinceSeconds=`date -d "-$numHours hour" +%s` ; req="[\"REQ\",\"l\",{\"since\":$sinceSeconds,\"limit\":$limit}]"; echo "echo $req | websocat $server | wc " ;