Netgoal: Update netgoal generate flags and variables

cmd/netgoal/generate.go

@@ -160,6 +160,9 @@ template modes for -t:`,
 			if nodeHostsToGenerate < 0 {
 				nodeHostsToGenerate = nodesToGenerate
 			}
+			if (nonPartnodesToGenerate >= 0) && (nonPartnodesHostsToGenerate == 0) {
+				nonPartnodesHostsToGenerate = nonPartnodesToGenerate
+			}
 			if relaysToGenerate < 0 {
 				reportErrorf("must specify number of relays with -R")
 			}

test/testdata/deployednettemplates/generate-recipe/generate_network.py

@@ -31,27 +31,29 @@ def build_netgoal_params(template_dict):
     instances = template_dict['instances']
 
     relay_count = 0
-    participating_node_count = 0
-    non_participating_node_count = 0
+    participating_instance_count = 0
+    non_participating_instance_count = 0
 
     for group in template_dict['groups']:
         relay_count += getInstanceCount(instances['relays'], group['percent']['relays'])
-        participating_node_count += getInstanceCount(instances['participatingNodes'], group['percent']['participatingNodes'])
-        non_participating_node_count += getInstanceCount(instances['nonParticipatingNodes'], group['percent']['nonParticipatingNodes'])
+        participating_instance_count += getInstanceCount(instances['participatingNodes'], group['percent']['participatingNodes'])
+        non_participating_instance_count += getInstanceCount(instances['nonParticipatingNodes'], group['percent']['nonParticipatingNodes'])
 
     relay_config = instances['relays']['config']
     participating_node_config = instances['participatingNodes']['config']
     non_participating_node_config = instances['nonParticipatingNodes']['config']
 
     wallets_count = template_dict['network']['wallets']
     nodes_count = template_dict['network']['nodes']
+    npn_count     =  template_dict['network']['npn']
 
     return [
         '-w', str(wallets_count),
         '-R', str(relay_count),
-        '-N', str(participating_node_count),
-        '-H', str(non_participating_node_count),
+        '-N', str(participating_instance_count),
+        '-H', str(non_participating_instance_count),
         '-n', str(nodes_count),
+        '-X', str(npn_count),
         '--relay-template', relay_config,
         '--node-template', participating_node_config,
         '--non-participating-node-template', non_participating_node_config
@@ -72,7 +74,7 @@ def build_genesis(template_path, netgoal_params, template_dict):
     ]
     args.extend(netgoal_params)
     netgoal(args, template_path)
-    if template_dict['network']['ConsensusProtocol']:
+    if 'ConsensusProtocol' in template_dict['network']:
         updateProtocol(f"{template_path}/generated/genesis.json", template_dict['network']['ConsensusProtocol'])
 
 def updateProtocol(genesis_path, consensus_protocol):

test/testdata/deployednettemplates/recipes/README.md

@@ -0,0 +1,45 @@
+# Recipes
+
+Most of the recipes' net.json and genesis.json use one of the following methods to call `netgoal generate`:
+1. `Makefile`
+2. `python3 {GO_ALGORAND_PATH}/test/testdata/deployednettemplates/generate-recipe/generate_network.py -f {PATH_TO}/network-tpl.json`
+
+Details for netgoal generate could be found in the binary with:
+```
+netgoal generate -h
+```
+
+Source code for netgoal can be found in `{GO_ALGORAND_PATH}/cmd/netgoal/generate.go`
+
+Quick reference when reading some of the Makefiles in this directory:
+```
+PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 -X 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+```
+- If you specify `-n` (number of participating algod nodes), the default if you don't pass in `-N` (number of machines to host the algod nodes) is set to match `-n`, i.e. one algod node per machine. To increase the number of nodes per machine, specify both `-n` and `-N`. `-n 10 -N 5` means 10 participating algod nodes across 5 machines, so 2 algod nodes per machine.
+- If you specify `-X` (number of non-participating nodes, npn), the default if you don't pass in `-H` (number of machines to host the npn) is set to the match `-X`, i.e. one npn per machine. To increase the number of nodes per machine, specify both `-X` and `-H`. `-X 10 -H 5` means 10 npns across 5 machines, so 2 npns per machine.
+
+```
+Usage:
+  netgoal generate [flags]
+
+Flags:
+      --bal stringArray                          Application Count
+  -h, --help                                     help for generate
+      --naccounts uint                           Account count (default 31)
+      --napps uint                               Application Count (default 7)
+      --nassets uint                             Asset count (default 5)
+  -N, --node-hosts int                           Node-hosts to generate, default=nodes (default -1)
+      --node-template string                     json for one node
+  -n, --nodes int                                Nodes to generate (default -1)
+      --non-participating-node-template string   json for non participating node
+  -X, --non-participating-nodes int              Non participating nodes to generate
+  -H, --non-participating-nodes-hosts int        Non participating nodes hosts to generate
+      --ntxns uint                               Transaction count (default 17)
+  -o, --outputfile string                        Output filename
+      --relay-template string                    json for a relay node
+  -R, --relays int                               Relays to generate (default -1)
+      --rounds uint                              Number of rounds (default 13)
+  -t, --template string                          Template to generate
+      --wallet-name string                       Source wallet name
+  -w, --wallets int                              Wallets to generate (default -1)
+```

test/testdata/deployednettemplates/recipes/alphanet-extension/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 20 -R 1 -N 20 -n 20 -H 1 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 20 -R 1 -N 20 -n 20 -H 1 -X 1 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	topology.json net.json genesis.json
 

test/testdata/deployednettemplates/recipes/alphanet/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 20 -R 5 -N 20 -n 20 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 20 -R 5 -N 20 -n 20 -H 10 -X 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	topology.json net.json genesis.json
 

test/testdata/deployednettemplates/recipes/bootstrappedScenario/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 -X 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 FILEPARAMS=--rounds 5000 --ntxns 1000 --naccounts 3000000 --nassets 20000 --napps 20000 --wallet-name "wallet1" --bal 100000 --bal 1000000
 
 all:	net.json genesis.json boostrappedFile.json

test/testdata/deployednettemplates/recipes/custom/example/network_templates/c5dmachines.json

@@ -2,6 +2,7 @@
   "network": {
     "wallets": 6,
     "nodes": 3,
+    "npn": 5,
     "ConsensusProtocol": "future"
   },
   "instances": {

test/testdata/deployednettemplates/recipes/custom/example/network_templates/network-tpl.json

@@ -2,6 +2,7 @@
   "network": {
     "wallets": 6,
     "nodes": 3,
+    "npn": 5,
     "ConsensusProtocol": "future"
   },
   "instances": {

test/testdata/deployednettemplates/recipes/custom/network_templates/network-tpl.json

@@ -1,24 +1,25 @@
 {
   "network": {
-    "wallets": 6,
-    "nodes": 3,
+    "wallets": 3,
+    "nodes": 1,
+    "npn": 1,
     "ConsensusProtocol": "future"
   },
   "instances": {
     "relays": {
       "config": "./configs/relay.json",
-      "type": "m5d.4xl",
+      "type": "m5d.2xl",
       "count": 1
     },
     "participatingNodes": {
       "config": "./configs/node.json",
-      "type": "m5d.4xl",
-      "count": 3
+      "type": "m5d.2xl",
+      "count": 1
     },
     "nonParticipatingNodes": {
       "config": "./configs/nonPartNode.json",
-      "type": "m5d.4xl",
-      "count": 5
+      "type": "m5d.2xl",
+      "count": 1
     }
   },
   "groups": [

test/testdata/deployednettemplates/recipes/mainnet-model/network-tpl.json

@@ -1,7 +1,8 @@
 {
   "network": {
     "wallets": 100,
-    "nodes": 50
+    "nodes": 50,
+    "npn": 10
   },
   "instances": {
     "relays": {

test/testdata/deployednettemplates/recipes/mmnet/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 130 -R 136 -n 130 -H 16 --node-template configs/node.json --relay-template configs/relay.json --non-participating-node-template configs/nonPartNode.json
+PARAMS=-w 130 -R 136 -n 130 -H 16 -X 16 --node-template configs/node.json --relay-template configs/relay.json --non-participating-node-template configs/nonPartNode.json
 
 all: topology.json net.json genesis.json
 

test/testdata/deployednettemplates/recipes/network-partition/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 -X 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json
 

test/testdata/deployednettemplates/recipes/scenario1/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 -X 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json
 

test/testdata/deployednettemplates/recipes/scenario1s/Makefile

@@ -1,5 +1,5 @@
 # scenario1s is scenario1 but smaller, (100 nodes, 100 wallets) -> (20 nodes, 20 wallets), each algod gets single tenancy on a smaller ec2 instance
-PARAMS=-w 20 -R 8 -N 20 -n 20 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 20 -R 8 -N 20 -n 20 -H 10 -X 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json
 

test/testdata/deployednettemplates/recipes/scenario2/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 400 -R 20 -N 40 -n 200 -H 15 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 400 -R 20 -N 40 -n 200 -H 15 -X 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json
 

test/testdata/deployednettemplates/recipes/scenario3/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 10000 -R 20 -N 100 -n 1000 -H 15 --node-template node.json --relay-template relay.json
+PARAMS=-w 10000 -R 20 -N 100 -n 1000 -H 15 -X 15 --node-template node.json --relay-template relay.json
 
 all:	net.json genesis.json
 

test/testdata/deployednettemplates/recipes/scenario3s/Makefile

@@ -1,5 +1,5 @@
 # scenario3s is scenario3 but smaller. (10000 wallets -> 500) (1000 algod participating nodes -> 100) It still keeps a global datacenter distribution.
-PARAMS=-w 500 -R 20 -N 100 -n 100 -H 15 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 500 -R 20 -N 100 -n 100 -H 15 -X 15 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 SOURCES=node.json ${GOPATH}/bin/netgoal Makefile relay.json nonPartNode.json
 

test/testdata/deployednettemplates/recipes/txnsync/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 10 -R 4 -N 10 -n 10 -H 1 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 10 -R 4 -N 10 -n 10 -H 1 -X 1 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json