Profiling documentation

docs/cli/profile.md

@@ -0,0 +1,42 @@
+## ⏱️ Profiling Models
+
+`nos profile` supports model benchmarking across a number of axes including iterations per second,
+memory footprint and utilization on CPU/GPU.
+Currently, the nos profiler itself runs natively in the execution environment (i.e. outside
+the NOS server), so you'll need to install both the `server` and `test` dependencies alongside
+your existing NOS installation.
+
+```bash
+pip install torch-nos[server,test]
+```
+
+## NOS Profile Commands
+
+### all
+Profile all models
+
+If you have the time, you can construct a profiling catalog on your machine in its entirety with:
+```bash
+nos profile all
+```
+### rebuild-catalog
+Generate a fresh catalog with 
+```bash
+nos profile rebuild-catalog
+```
+### model
+You can also profile specific models with 
+```bash
+nos profile model -m openai/clip-vit-large-patch14
+```
+### method
+Or an entire method type with
+```bash
+nos profile method -m encode_image
+```
+to benchmark e.g. all image embedding models.
+
+### list
+Dump the nos profiling catalog with `nos profile list`
+
+![NOS Profile List](../assets/nos_catalog_t4_complete.png)

nos/common/spec.py

@@ -433,19 +433,30 @@ def load(self, model_method_id: Any) -> "ModelSpec":
 
     def load_profile_catalog(self) -> "ModelSpecMetadataCatalog":
         """Load the model profiles from a JSON catalog."""
+        import logging
+        import os
+        from pathlib import Path
+
         import pandas as pd
 
-        if not NOS_PROFILE_CATALOG_PATH.exists():
-            raise FileNotFoundError(f"Model metadata catalog not found, path={NOS_PROFILE_CATALOG_PATH}.")
+        logger = logging.getLogger(__name__)
 
-        import logging
+        catalog_path = NOS_PROFILE_CATALOG_PATH
 
-        logger = logging.getLogger(__name__)
-        debug_str = "Loading profiling catalog from " + str(NOS_PROFILE_CATALOG_PATH)
+        # Check if we have NOS_PROFILE_CATALOG_PATH_OVERRIDE in the environment
+        if os.environ.get("NOS_PROFILE_CATALOG_PATH_OVERRIDE"):
+            catalog_path = Path(os.environ["NOS_PROFILE_CATALOG_PATH_OVERRIDE"])
+            logger.debug(f"Using custom profile catalog [path={catalog_path}]")
+
+        if not catalog_path.exists():
+            # Make sure the catalog (either default or custom) exists
+            raise FileNotFoundError(f"Model metadata catalog not found, path={catalog_path}.")
+
+        debug_str = "Loading profiling catalog from " + str(catalog_path)
         logger.info(debug_str)
 
         # Read the catalog
-        df = pd.read_json(str(NOS_PROFILE_CATALOG_PATH), orient="records")
+        df = pd.read_json(str(catalog_path), orient="records")
         columns = df.columns
         # Check if the catalog is valid with the required columns