more changes

skylark/benchmark/pareto_speedups.py

@@ -29,11 +29,11 @@ def benchmark(
         required_throughput_gbits=min_throughput * instance_limit,
         gbyte_to_transfer=gbyte_to_transfer,
         instance_limit=instance_limit,
-        sparsity_penalty=sparsity_penalty,
-        max_connections_per_path=max_connections_per_path,
+        unused_sparsity_penalty=sparsity_penalty,
+        benchmark_throughput_connections=max_connections_per_path,
         max_connections_per_node=max_connections_per_node,
         solver=cp.GUROBI,
-        solver_verbose=False
+        solver_verbose=False,
     )
     if solution["feasible"]:
         baseline_throughput = solver.get_path_throughput(src, dst) / GB
@@ -71,15 +71,17 @@ def main(args):
             if src != dst:
                 for instance_limit in [1, 2, 4]:
                     for min_throughput in np.linspace(0, args.max_throughput * instance_limit, args.num_throughputs)[1:]:
-                        configs.append(dict(
-                            src=src,
-                            dst=dst,
-                            min_throughput=min_throughput,
-                            gbyte_to_transfer=args.gbyte_to_transfer,
-                            instance_limit=instance_limit,
-                            max_connections_per_path=64,
-                            max_connections_per_node=64,
-                        ))
+                        configs.append(
+                            dict(
+                                src=src,
+                                dst=dst,
+                                min_throughput=min_throughput,
+                                gbyte_to_transfer=args.gbyte_to_transfer,
+                                instance_limit=instance_limit,
+                                max_connections_per_path=64,
+                                max_connections_per_node=64,
+                            )
+                        )
 
     results = []
     for config in tqdm(configs, desc="dispatch"):

skylark/cli/cli_solver.py

@@ -49,7 +49,7 @@ def solve_throughput(
         required_throughput_gbits=required_throughput_gbits,
         gbyte_to_transfer=gbyte_to_transfer,
         instance_limit=max_instances,
-        sparsity_penalty=sparsity_penalty,
+        unused_sparsity_penalty=sparsity_penalty,
         solver=solver,
         solver_verbose=solver_verbose,
         save_lp_path=skylark_root / "data" / "throughput_solver.lp",

skylark/replicate/solver.py

@@ -94,9 +94,10 @@ def solve_min_cost(
         aws_instance_throughput_limit=5.0,
         gcp_instance_throughput_limit=8.0,
         azure_instance_throughput_limit=16.0,
-        max_connections_per_path=64,
+        benchmark_throughput_connections=64,
         max_connections_per_node=64,
-        sparsity_penalty=5e-3,
+        unused_sparsity_penalty=5e-3,
+        cost_per_instance_hr=1.54,  # m5.8xlarge
         solver=cp.GLPK,
         solver_verbose=False,
         save_lp_path=None,
@@ -111,39 +112,35 @@ def solve_min_cost(
         edge_cost_per_gigabyte, edge_capacity_gigabits = self.get_cost_grid(), self.get_throughput_grid()
 
         # define variables
-        edge_flow_gigabits = cp.Variable((len(regions), len(regions)), name="edge_flow_gigabits")
+        edge_flow_gigabits_per_conn = cp.Variable((len(regions), len(regions)), name="edge_flow_gigabits_per_conn")
         conn = cp.Variable((len(regions), len(regions)), name="conn")
-        inv_n_instances = cp.Variable(name="n_instances")  # integer=True
-        n_instances = cp.inv_pos(inv_n_instances)
+        edge_flow_gigabits = cp.multiply(edge_flow_gigabits_per_conn, conn)
+        instances_per_region = cp.Variable((len(regions)), name="instances_per_region")  # integer=True
         node_flow_in = cp.sum(edge_flow_gigabits, axis=0)
         node_flow_out = cp.sum(edge_flow_gigabits, axis=1)
-        req_tput_instance = required_throughput_gbits * inv_n_instances
 
         constraints = []
 
+        # instance limit
+        constraints.append(instances_per_region <= instance_limit)
+        constraints.append(instances_per_region >= 0)
+
         # connection limits
         constraints.append(conn >= 0)
-        constraints.append(conn <= max_connections_per_path)
-        node_conns_out = cp.sum(conn, axis=1)
-        constraints.append(node_conns_out <= max_connections_per_node)
-
-        # instance limit
-        constraints.append(inv_n_instances >= 1.0 / instance_limit)
-        constraints.append(inv_n_instances <= 1)
-        # constraints.append(n_instances >= 1)
+        constraints.append(cp.sum(conn, axis=1) <= max_connections_per_node * instances_per_region)
 
         # flow capacity constraint
-        adjusted_edge_capacity_gigabits = cp.multiply(edge_capacity_gigabits, conn / max_connections_per_path)
+        adjusted_edge_capacity_gigabits = cp.multiply(edge_capacity_gigabits, conn / benchmark_throughput_connections)
         constraints.append(edge_flow_gigabits <= adjusted_edge_capacity_gigabits)
 
         # flow conservation
         for v in range(len(regions)):
             f_in, f_out = node_flow_in[v], node_flow_out[v]
             if v in sources:
                 constraints.append(f_in == 0)
-                constraints.append(f_out == req_tput_instance)
+                constraints.append(f_out == required_throughput_gbits)
             elif v in sinks:
-                constraints.append(f_in == req_tput_instance)
+                constraints.append(f_in == required_throughput_gbits)
             else:
                 constraints.append(f_in == f_out)
 
@@ -155,11 +152,11 @@ def solve_min_cost(
             provider = r.split(":")[0]
             f_out = node_flow_out[idx]
             if provider == "aws":
-                constraints.append(f_out <= aws_instance_throughput_limit)
+                constraints.append(f_out <= aws_instance_throughput_limit * instances_per_region[idx])
             elif provider == "gcp":
-                constraints.append(f_out <= gcp_instance_throughput_limit)
+                constraints.append(f_out <= gcp_instance_throughput_limit * instances_per_region[idx])
             elif provider == "azure":
-                constraints.append(f_out <= azure_instance_throughput_limit)
+                constraints.append(f_out <= azure_instance_throughput_limit * instances_per_region[idx])
             else:
                 raise ValueError(f"Unknown provider {provider}")
 
@@ -172,14 +169,15 @@ def solve_min_cost(
         cost_per_edge = cp.multiply(edge_flow_gigabits * runtime_s, edge_cost_per_gigabit)  #  gbit/s * $/gbit = $/s
         total_cost = cp.sum(cost_per_edge)
 
+        # instance cost
+        per_instance_cost = cost_per_instance_hr / 3600 * runtime_s
+        instance_cost = cp.sum(instances_per_region) * per_instance_cost
+
         # sparsity constraint
-        # connnection constraint avoids using overlay versus direct path as an example
-        sparsity_constraint = cp.norm1(conn)
-        # todo add small penalty for using more instances than needed (prefer more compact solutions)
-        # e.g. 1x64 connections direct versus 2x32 connections direct
+        # todo add small penalty for using more instances than needed (prefer more compact solutions) e.g. 1x64 connections direct versus 2x32 connections direct
         # todo just make this an ILP avoid these sparsity constraints
 
-        prob = cp.Problem(cp.Minimize(total_cost + sparsity_penalty * sparsity_constraint), constraints)
+        prob = cp.Problem(cp.Minimize(total_cost), constraints)  #  + instance_cost
 
         if solver == cp.GUROBI or solver == "gurobi":
             solver_options = {}