Netgoal: Update netgoal generate flags and variables

cmd/netgoal/README.md

@@ -0,0 +1,52 @@
+# Netgoal
+
+## netgoal generate
+`--participation-host-machines (-N)` and `--npn-host-machines (-X)` are optional parameters and they default to `--participation-algod-nodes (-n)` and `--npn-algod-nodes (-x)` respectively, i.e. defaults to a machine per algod node.
+
+### Long-Form Flags Example
+- Wallets: The command below will generate 100 wallets for the 100 participation algod nodes. By default each npn gets one wallet each. If there are more wallets than nodes, it will get split across the participation algod nodes.
+- Relays: 8 Relays and 8 machines to host the relays will be generated
+- Participation Nodes: 100 particiipation algod nodes will be distributed across 20 host machines.
+- Non-Participation Nodes (NPNs): 10 non-participation algod nodes will be distributed across 5 host machines.
+
+```
+netgoal generate -t net -r /tmp/wat -o net.json --wallets 100 --relays 8 --participation-host-machines 20 --participation-algod-nodes 100 --npn-host-machines 5 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+```
+
+### Short-Form Flags Example
+The following will result in the same outcome as the command above.
+```
+netgoal generate -t net -r /tmp/wat -o net.json -w 100 -R 8 -N 20 -n 100 -X 5 -x 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+```
+## Flags
+```
+netgoal generate -h
+
+Usage:
+  netgoal generate [flags]
+
+Flags:
+      --bal stringArray                          Application Count
+  -h, --help                                     help for generate
+      --naccounts uint                           Account count (default 31)
+      --napps uint                               Application Count (default 7)
+      --nassets uint                             Asset count (default 5)
+      --node-template string                     json for one node
+      --non-participating-node-template string   json for non participating node
+  -x, --npn-algod-nodes int                      Total non-participation algod nodes to generate
+  -X, --npn-host-machines int                    Host machines to generate for non-participation algod nodes, default=npn-algod-nodes
+      --ntxns uint                               Transaction count (default 17)
+  -o, --outputfile string                        Output filename
+  -n, --participation-algod-nodes int            Total participation algod nodes to generate (default -1)
+  -N, --participation-host-machines int          Host machines to generate for participation algod nodes, default=participation-algod-nodes (default -1)
+      --relay-template string                    json for a relay node
+  -R, --relays int                               Relays to generate (default -1)
+      --rounds uint                              Number of rounds (default 13)
+  -t, --template string                          Template to generate
+      --wallet-name string                       Source wallet name
+  -w, --wallets int                              Wallets to generate (default -1)
+
+Global Flags:
+  -m, --modifier string   Override Genesis Version Modifier (eg 'v1')
+  -r, --rootdir string    Root directory for the private network directories
+```

cmd/netgoal/generate.go

@@ -37,10 +37,10 @@ import (
 var outputFilename string
 var templateToGenerate string
 var relaysToGenerate int
-var nodesToGenerate int
-var nodeHostsToGenerate int
-var nonPartnodesToGenerate int
-var nonPartnodesHostsToGenerate int
+var participationAlgodNodes int
+var participationHostMachines int
+var npnAlgodNodes int
+var npnHostMachines int
 var walletsToGenerate int
 var nodeTemplatePath string
 var nonParticipatingNodeTemplatePath string
@@ -63,10 +63,10 @@ func init() {
 
 	generateCmd.Flags().IntVarP(&walletsToGenerate, "wallets", "w", -1, "Wallets to generate")
 	generateCmd.Flags().IntVarP(&relaysToGenerate, "relays", "R", -1, "Relays to generate")
-	generateCmd.Flags().IntVarP(&nodeHostsToGenerate, "node-hosts", "N", -1, "Node-hosts to generate, default=nodes")
-	generateCmd.Flags().IntVarP(&nodesToGenerate, "nodes", "n", -1, "Nodes to generate")
-	generateCmd.Flags().IntVarP(&nonPartnodesToGenerate, "non-participating-nodes", "X", 0, "Non participating nodes to generate")
-	generateCmd.Flags().IntVarP(&nonPartnodesHostsToGenerate, "non-participating-nodes-hosts", "H", 0, "Non participating nodes hosts to generate")
+	generateCmd.Flags().IntVarP(&participationAlgodNodes, "participation-algod-nodes", "n", -1, "Total participation algod nodes to generate")
+	generateCmd.Flags().IntVarP(&participationHostMachines, "participation-host-machines", "N", -1, "Host machines to generate for participation algod nodes, default=participation-algod-nodes")
+	generateCmd.Flags().IntVarP(&npnAlgodNodes, "npn-algod-nodes", "x", 0, "Total non-participation algod nodes to generate")
+	generateCmd.Flags().IntVarP(&npnHostMachines, "npn-host-machines", "X", 0, "Host machines to generate for non-participation algod nodes, default=npn-algod-nodes")
 	generateCmd.Flags().StringVarP(&nodeTemplatePath, "node-template", "", "", "json for one node")
 	generateCmd.Flags().StringVarP(&nonParticipatingNodeTemplatePath, "non-participating-node-template", "", "", "json for non participating node")
 	generateCmd.Flags().StringVarP(&relayTemplatePath, "relay-template", "", "", "json for a relay node")
@@ -149,24 +149,27 @@ template modes for -t:`,
 			if walletsToGenerate < 0 {
 				reportErrorf("must specify number of wallets with -w")
 			}
-			err = generateWalletGenesis(outputFilename, walletsToGenerate, nonPartnodesToGenerate)
+			err = generateWalletGenesis(outputFilename, walletsToGenerate, npnAlgodNodes)
 		case "net", "network", "goalnet":
 			if walletsToGenerate < 0 {
 				reportErrorf("must specify number of wallets with -w")
 			}
-			if nodesToGenerate < 0 {
+			if participationAlgodNodes < 0 {
 				reportErrorf("must specify number of nodes with -n")
 			}
-			if nodeHostsToGenerate < 0 {
-				nodeHostsToGenerate = nodesToGenerate
+			if participationHostMachines < 0 {
+				participationHostMachines = participationAlgodNodes
+			}
+			if (npnAlgodNodes >= 0) && (npnHostMachines == 0) {
+				npnHostMachines = npnAlgodNodes
 			}
 			if relaysToGenerate < 0 {
 				reportErrorf("must specify number of relays with -R")
 			}
 			if templateType == "goalnet" {
-				err = generateNetworkGoalTemplate(outputFilename, walletsToGenerate, relaysToGenerate, nodesToGenerate, nonPartnodesHostsToGenerate)
+				err = generateNetworkGoalTemplate(outputFilename, walletsToGenerate, relaysToGenerate, participationAlgodNodes, npnAlgodNodes)
 			} else {
-				err = generateNetworkTemplate(outputFilename, walletsToGenerate, relaysToGenerate, nodeHostsToGenerate, nodesToGenerate, nonPartnodesHostsToGenerate, nonPartnodesToGenerate, baseNode, baseNonParticipatingNode, baseRelay)
+				err = generateNetworkTemplate(outputFilename, walletsToGenerate, relaysToGenerate, participationHostMachines, participationAlgodNodes, npnHostMachines, npnAlgodNodes, baseNode, baseNonParticipatingNode, baseRelay)
 			}
 		case "otwt":
 			err = generateNetworkTemplate(outputFilename, 1000, 10, 20, 100, 0, 0, baseNode, baseNonParticipatingNode, baseRelay)
@@ -234,9 +237,9 @@ func pickNodeConfig(alt []remote.NodeConfig, name string) remote.NodeConfig {
 	return alt[0]
 }
 
-func generateNetworkGoalTemplate(templateFilename string, wallets, relays, nodes, npnHosts int) error {
+func generateNetworkGoalTemplate(templateFilename string, wallets, relays, nodes, npnNodes int) error {
 	template := netdeploy.NetworkTemplate{}
-	template.Nodes = make([]remote.NodeConfigGoal, 0, relays+nodes+npnHosts)
+	template.Nodes = make([]remote.NodeConfigGoal, 0, relays+nodes+npnNodes)
 	template.Genesis = generateWalletGenesisData(walletsToGenerate, 0)
 	for i := 0; i < relays; i++ {
 		name := "relay" + strconv.Itoa(i+1)
@@ -257,7 +260,7 @@ func generateNetworkGoalTemplate(templateFilename string, wallets, relays, nodes
 		template.Nodes = append(template.Nodes, newNode)
 	}
 
-	for i := 0; i < npnHosts; i++ {
+	for i := 0; i < npnNodes; i++ {
 		name := "nonParticipatingNode" + strconv.Itoa(i+1)
 		newNode := remote.NodeConfigGoal{
 			Name:    name,
@@ -286,8 +289,8 @@ func generateNetworkGoalTemplate(templateFilename string, wallets, relays, nodes
 		}
 	}
 
-	if npnHosts > 0 {
-		for walletIndex < npnHosts {
+	if npnNodes > 0 {
+		for walletIndex < npnNodes {
 			for nodei, node := range template.Nodes {
 				if node.Name[0:4] != "nonP" {
 					continue
@@ -298,11 +301,11 @@ func generateNetworkGoalTemplate(templateFilename string, wallets, relays, nodes
 				}
 				template.Nodes[nodei].Wallets = append(template.Nodes[nodei].Wallets, wallet)
 				walletIndex++
-				if walletIndex >= npnHosts {
+				if walletIndex >= npnNodes {
 					break
 				}
 			}
-			if walletIndex >= npnHosts {
+			if walletIndex >= npnNodes {
 				break
 			}
 		}
@@ -478,18 +481,18 @@ func saveGoalTemplateToDisk(template netdeploy.NetworkTemplate, filename string)
 	return err
 }
 
-func generateWalletGenesisData(wallets, npnHosts int) gen.GenesisData {
+func generateWalletGenesisData(wallets, npnNodes int) gen.GenesisData {
 	ratZero := big.NewRat(int64(0), int64(1))
 	ratHundred := big.NewRat(int64(100), int64(1))
 	data := gen.DefaultGenesis
-	totalWallets := wallets + npnHosts
+	totalWallets := wallets + npnNodes
 	data.Wallets = make([]gen.WalletData, totalWallets)
 	participatingNodeStake := big.NewRat(int64(100), int64(wallets))
 	nonParticipatingNodeStake := ratZero
-	if npnHosts > 0 {
+	if npnNodes > 0 {
 		// split participating an non participating stake evenly
 		participatingNodeStake = big.NewRat(int64(50), int64(wallets))
-		nonParticipatingNodeStake = big.NewRat(int64(50), int64(npnHosts))
+		nonParticipatingNodeStake = big.NewRat(int64(50), int64(npnNodes))
 	}
 
 	stake := ratZero
@@ -519,8 +522,8 @@ func generateWalletGenesisData(wallets, npnHosts int) gen.GenesisData {
 	return data
 }
 
-func generateWalletGenesis(filename string, wallets, npnHosts int) error {
-	data := generateWalletGenesisData(wallets, npnHosts)
+func generateWalletGenesis(filename string, wallets, npnNodes int) error {
+	data := generateWalletGenesisData(wallets, npnNodes)
 	return saveGenesisDataToDisk(data, filename)
 }
 

test/testdata/deployednettemplates/generate-recipe/generate_network.py

@@ -31,27 +31,29 @@ def build_netgoal_params(template_dict):
     instances = template_dict['instances']
 
     relay_count = 0
-    participating_node_count = 0
-    non_participating_node_count = 0
+    participating_instance_count = 0
+    non_participating_instance_count = 0
 
     for group in template_dict['groups']:
         relay_count += getInstanceCount(instances['relays'], group['percent']['relays'])
-        participating_node_count += getInstanceCount(instances['participatingNodes'], group['percent']['participatingNodes'])
-        non_participating_node_count += getInstanceCount(instances['nonParticipatingNodes'], group['percent']['nonParticipatingNodes'])
+        participating_instance_count += getInstanceCount(instances['participatingNodes'], group['percent']['participatingNodes'])
+        non_participating_instance_count += getInstanceCount(instances['nonParticipatingNodes'], group['percent']['nonParticipatingNodes'])
 
     relay_config = instances['relays']['config']
     participating_node_config = instances['participatingNodes']['config']
     non_participating_node_config = instances['nonParticipatingNodes']['config']
 
     wallets_count = template_dict['network']['wallets']
     nodes_count = template_dict['network']['nodes']
+    npn_count     =  template_dict['network']['npn']
 
     return [
         '-w', str(wallets_count),
         '-R', str(relay_count),
-        '-N', str(participating_node_count),
-        '-H', str(non_participating_node_count),
+        '-N', str(participating_instance_count),
+        '-X', str(non_participating_instance_count),
         '-n', str(nodes_count),
+        '-x', str(npn_count),
         '--relay-template', relay_config,
         '--node-template', participating_node_config,
         '--non-participating-node-template', non_participating_node_config
@@ -72,7 +74,7 @@ def build_genesis(template_path, netgoal_params, template_dict):
     ]
     args.extend(netgoal_params)
     netgoal(args, template_path)
-    if template_dict['network']['ConsensusProtocol']:
+    if 'ConsensusProtocol' in template_dict['network']:
         updateProtocol(f"{template_path}/generated/genesis.json", template_dict['network']['ConsensusProtocol'])
 
 def updateProtocol(genesis_path, consensus_protocol):

test/testdata/deployednettemplates/recipes/README.md

@@ -0,0 +1,19 @@
+# Recipes
+
+Most of the recipes' net.json and genesis.json use one of the following methods to call `netgoal generate`:
+1. `Makefile`
+2. `python3 {GO_ALGORAND_PATH}/test/testdata/deployednettemplates/generate-recipe/generate_network.py -f {PATH_TO}/network-tpl.json`
+
+Details for netgoal generate could be found in the binary with:
+```
+netgoal generate -h
+```
+
+Source code for netgoal can be found in `{GO_ALGORAND_PATH}/cmd/netgoal/generate.go`
+Documentation: https://github.com/algorand/go-algorand/tree/master/cmd/netgoal/README.md
+
+Make sure you set the PATH and GOPATH variables to the netgoal binary's path.
+
+## Custom Recipe
+Leverages the generate_network.py script and has unique instructions found in the README:
+https://github.com/algorand/go-algorand/tree/master/test/testdata/deployednettemplates/recipes/custom

test/testdata/deployednettemplates/recipes/alphanet-extension/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 20 -R 1 -N 20 -n 20 -H 1 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 20 -R 1 -N 20 -n 20 --npn-algod-nodes 1 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	topology.json net.json genesis.json
 

test/testdata/deployednettemplates/recipes/alphanet/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 20 -R 5 -N 20 -n 20 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 20 -R 5 -N 20 -n 20 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	topology.json net.json genesis.json
 

test/testdata/deployednettemplates/recipes/betanet-model-2/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 20 -R 5 -N 20 -n 20 -H 20 -X 20 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 20 -R 5 -N 20 -n 20 --npn-algod-nodes 20 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	topology.json net.json genesis.json
 

test/testdata/deployednettemplates/recipes/bootstrappedScenario/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 100 -R 8 -N 20 -n 100 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 FILEPARAMS=--rounds 5000 --ntxns 1000 --naccounts 3000000 --nassets 20000 --napps 20000 --wallet-name "wallet1" --bal 100000 --bal 1000000
 
 all:	net.json genesis.json boostrappedFile.json

test/testdata/deployednettemplates/recipes/custom/README.md

@@ -22,7 +22,7 @@ Build and create the recipe.
 
 ## "Quick" Start - Manual recipe generation (not using Jenkins)
 Generate the recipe with the `network-tpl.json` file
-- (See the first section above for small networks.)
+- (See the first section above for small networks. See Troubleshooting for netgoal path set up)
 1. Make sure you're in the same directory as this README and `cp network_templates/network-tpl.json network-tpl.json`
 2. Generate the recipe with a python script:
 ```
@@ -67,7 +67,7 @@ Most parameters that can be modified by config.json can be found in `go-algorand
 
 ## Troubleshooting
 ### Can't find netgoal
-- Make sure you have netgoal installed
+- Make sure you have netgoal installed (you can either download it or run through the go-algorand build process)
 - Make sure you export GOBIN and GOPATH in your environment and add it to your path.
 On a mac, update by editing `~/.zshrc`, add
 ```
@@ -81,4 +81,4 @@ export PATH=$PATH:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/Users/ec2-user/L
 - Make sure the machine type exists. It uses the regions in the groups and the type to come up with the host template name in `test/testdata/deployednettemplates/hosttemplates/hosttemplates.json`. If it doesn't exist, you will have to add it to that file.
 
 ### couldn't initialize the node: unsupported protocol
-- check your consensus.json. It may be missing the keys in the future protocol if you are doing this manually. Compare the consensus.json with `goal protocols > generated_consensus.json`
+- check your consensus.json. It may be missing the keys in the future protocol if you are doing this manually. Compare the consensus.json with `goal protocols > generated_consensus.json`

test/testdata/deployednettemplates/recipes/custom/example/network_templates/c5dmachines.json

@@ -2,6 +2,7 @@
   "network": {
     "wallets": 6,
     "nodes": 3,
+    "npn": 5,
     "ConsensusProtocol": "future"
   },
   "instances": {

test/testdata/deployednettemplates/recipes/custom/example/network_templates/network-tpl.json

@@ -2,6 +2,7 @@
   "network": {
     "wallets": 6,
     "nodes": 3,
+    "npn": 5,
     "ConsensusProtocol": "future"
   },
   "instances": {

test/testdata/deployednettemplates/recipes/custom/network_templates/network-tpl.json

@@ -1,24 +1,25 @@
 {
   "network": {
-    "wallets": 6,
-    "nodes": 3,
+    "wallets": 3,
+    "nodes": 1,
+    "npn": 1,
     "ConsensusProtocol": "future"
   },
   "instances": {
     "relays": {
       "config": "./configs/relay.json",
-      "type": "m5d.4xl",
+      "type": "m5d.2xl",
       "count": 1
     },
     "participatingNodes": {
       "config": "./configs/node.json",
-      "type": "m5d.4xl",
-      "count": 3
+      "type": "m5d.2xl",
+      "count": 1
     },
     "nonParticipatingNodes": {
       "config": "./configs/nonPartNode.json",
-      "type": "m5d.4xl",
-      "count": 5
+      "type": "m5d.2xl",
+      "count": 1
     }
   },
   "groups": [

test/testdata/deployednettemplates/recipes/mainnet-model/network-tpl.json

@@ -1,7 +1,8 @@
 {
   "network": {
     "wallets": 100,
-    "nodes": 50
+    "nodes": 50,
+    "npn": 10
   },
   "instances": {
     "relays": {

test/testdata/deployednettemplates/recipes/mmnet/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 130 -R 136 -n 130 -H 16 --node-template configs/node.json --relay-template configs/relay.json --non-participating-node-template configs/nonPartNode.json
+PARAMS=-w 130 -R 136 -n 130 --npn-algod-nodes 16 --node-template configs/node.json --relay-template configs/relay.json --non-participating-node-template configs/nonPartNode.json
 
 all: topology.json net.json genesis.json
 

test/testdata/deployednettemplates/recipes/network-partition/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 100 -R 8 -N 20 -n 100 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json
 

test/testdata/deployednettemplates/recipes/scenario1/Makefile

@@ -1,4 +1,4 @@
-PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 100 -R 8 -N 20 -n 100 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json
 

test/testdata/deployednettemplates/recipes/scenario1s/Makefile

@@ -1,5 +1,5 @@
 # scenario1s is scenario1 but smaller, (100 nodes, 100 wallets) -> (20 nodes, 20 wallets), each algod gets single tenancy on a smaller ec2 instance
-PARAMS=-w 20 -R 8 -N 20 -n 20 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
+PARAMS=-w 20 -R 8 -N 20 -n 20 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
 
 all:	net.json genesis.json