More

coiled · Feb 17, 2024 · d3d5bd7 · d3d5bd7
1 parent 13a2270
commit d3d5bd7
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/pipeline/reduce.py b/pipeline/reduce.py
@@ -27,7 +27,7 @@ def save_query(region, part_type):
             coiled.Cluster,
             name="reduce",
             region=REGION,
-            n_workers=20,
+            n_workers=10,
             tags={"workflow": "etl-tpch"},
             shutdown_on_close=False,
             idle_timeout="5 minutes",

diff --git a/pipeline/train.py b/pipeline/train.py
@@ -31,7 +31,8 @@ def train(file):
     with tempfile.TemporaryDirectory() as tmpdir:
         out = Path(tmpdir) / "model.json"
         model.save_model(out)
-        fs.mv(str(out), str(MODEL_FILE.parent))
+        fs.makedirs(MODEL_FILE.parent, exist_ok=True)
+        fs.put(out, MODEL_FILE)
     return model