Rebase profiling

Signed-off-by: Vishnu Challa <[email protected]>

Makefile

@@ -51,12 +51,8 @@ run: ## Run the service locally
 memray-run: ## Run the service locally using memray
 	python -m memray run -o ./memory_profile.bin runner.py
 
-memray-flamegraph: ## Extract flamegraph from memray output
-	@if [ -f ./memory_profile.bin ]; then \
-		python -m memray flamegraph ./memory_profile.bin; \
-	else \
-		echo "./memory_profile.bin doesn't exist. please restart the service using make memray-run"; \
-	fi
+memray-flamegraph: memray-run
+	python -m memray flamegraph ./memory_profile.bin
 
 test: test-unit test-integration test-e2e ## Run all tests
 

README.md

@@ -425,6 +425,23 @@ By default this interface will ask the OLS server to retain and use your convers
 
 OLS API documentation is available at http://localhost:8080/docs
 
+###  CPU profiling
+To enable CPU profiling, please deploy your own pyroscope server and specify its URL in the `devconfig` as shown below. This will help OLS to send profiles to a specified endpoint.
+
+```yaml
+dev_config:
+  pyroscope_url: https://your-pyroscope-url.com
+```
+
+### Memory profiling
+To enable memory profiling, simply start the server with the below command.
+```
+make memray-run
+```
+Once you are done executing a few queries and want to look at the memory flamegraphs, please run the below command and it should spit out a html file for us.
+```
+make memray-flamegraph
+```
 
 ## Deploying OLS on OpenShift
 

docs/config.puml

@@ -34,6 +34,7 @@ class "DevConfig" as ols.app.models.config.DevConfig {
   disable_auth : bool
   disable_tls : bool
   enable_dev_ui : bool
+  pyroscope_url: Optional[str]
   k8s_auth_token : Optional[str]
   llm_params : dict
   run_on_localhost : bool

examples/olsconfig-local-ollama.yaml

@@ -42,6 +42,7 @@ dev_config:
   enable_dev_ui: true
   disable_auth: true
   disable_tls: true
+  pyroscope_url: "https://pyroscope.pyroscope.svc.cluster.local:4040"
   # llm_params:
   #   temperature_override: 0
   # k8s_auth_token: optional_token_when_no_available_kube_config

examples/olsconfig.yaml

@@ -88,6 +88,7 @@ dev_config:
   enable_dev_ui: true
   disable_auth: true
   disable_tls: true
+  pyroscope_url: "https://pyroscope.pyroscope.svc.cluster.local:4040"
   # llm_params:
   #   temperature_override: 0
   # k8s_auth_token: optional_token_when_no_available_kube_config

examples/openshift-lightspeed-tls.yaml

@@ -230,6 +230,7 @@ data:
         transcripts_disabled: true
     dev_config:
       enable_dev_ui: true
+      pyroscope_url: https://pyroscope.pyroscope.svc.cluster.local:4040
       # llm_temperature_override: 0
       disable_auth: true
 immutable: false

ols/app/models/config.py

@@ -955,6 +955,7 @@ class DevConfig(BaseModel):
     llm_params: dict = {}
     disable_auth: bool = False
     disable_tls: bool = False
+    pyroscope_url: Optional[str] = None
     k8s_auth_token: Optional[str] = None
     run_on_localhost: bool = False