Working SCITT receipts with scitt-community/scitt-api-emulator@1e4ec88

Signed-off-by: John Andersen <[email protected]>

config.yaml

@@ -7,5 +7,6 @@ litellm_settings:
   callbacks: scitt_validated_tool_use.LiteLLMSCITTValidatedToolUse
   callback_params_cls: scitt_validated_tool_use.LiteLLMSCITTValidatedToolUseParams
   callback_params:
-    scrapi_instance_urls:
-    - 'https://scitt.unstable.chadig.com'
+    scrapi_instances:
+    # - url: 'https://scitt.unstable.chadig.com'
+    - url: 'http://localhost:8000'

litellm/integrations/custom_logger.py

@@ -6,7 +6,7 @@
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.caching import DualCache
 
-from typing import Literal, Union
+from typing import Literal, Union, AsyncIterator, Any
 
 dotenv.load_dotenv()  # Loading env variables using dotenv
 import traceback
@@ -82,7 +82,7 @@ async def async_moderation_hook(
     async def async_post_call_streaming_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
-        response: str,
+        chunk: Any,
     ):
         pass
 

litellm/main.py

@@ -7,12 +7,13 @@
 #
 #  Thank you ! We ❤️ you! - Krrish & Ishaan
 
-import os, openai, sys, json, inspect, uuid, datetime, threading, tempfile, pathlib
+import os, openai, sys, json, inspect, uuid, datetime, threading
 from typing import Any, Literal, Union, BinaryIO
 from functools import partial
 import dotenv, traceback, random, asyncio, time, contextvars
 from copy import deepcopy
 
+import httpx
 import litellm
 from ._logging import verbose_logger
 from litellm import (  # type: ignore

litellm/proxy/proxy_server.py

@@ -1658,8 +1658,6 @@ def get_callback_params_kwarg(callback_cls, callback_params_cls):
     raise Exception(f"Could not find an Optional[{callback_params_cls}] keyword argument in {callback_cls}.__init__(**kwargs)")
 
 
-import snoop
-@snoop
 def callback_instance(
     litellm_settings,
     value,
@@ -2659,13 +2657,9 @@ async def async_data_generator(response, user_api_key_dict):
     verbose_proxy_logger.debug("inside generator")
     try:
         start_time = time.time()
-        async for chunk in response:
-            ### CALL HOOKS ### - modify outgoing data
-            with snoop():
-                chunk = await proxy_logging_obj.post_call_streaming_hook(
-                    user_api_key_dict=user_api_key_dict,
-                    chunk=chunk,
-                )
+        async for chunk in proxy_logging_obj.post_call_streaming_hook(
+            response, user_api_key_dict,
+        ):
             chunk = chunk.model_dump_json(exclude_none=True)
             try:
                 yield f"data: {chunk}\n\n"
@@ -3283,10 +3277,9 @@ async def chat_completion(
         fastapi_response.headers["x-litellm-model-id"] = model_id
 
         ### CALL HOOKS ### - modify outgoing data
-        with snoop():
-            response = await proxy_logging_obj.post_call_success_hook(
-                user_api_key_dict=user_api_key_dict, response=response
-            )
+        response = await proxy_logging_obj.post_call_success_hook(
+            user_api_key_dict=user_api_key_dict, response=response
+        )
 
         return response
     except Exception as e:

litellm/proxy/utils.py

@@ -1,4 +1,4 @@
-from typing import Optional, List, Any, Literal, Union
+from typing import Optional, List, Any, Literal, Union, AsyncIterator
 import os, subprocess, hashlib, importlib, asyncio, copy, json, aiohttp, httpx
 import litellm, backoff
 from litellm.proxy._types import (
@@ -447,10 +447,9 @@ async def post_call_success_hook(
         1. /chat/completions
         """
         new_response = copy.deepcopy(response)
-        snoop.pp(litellm.callbacks)
         for callback in litellm.callbacks:
             try:
-                # This doesn't work with spec style import
+                # isinstance doesn't work with module spec imports
                 if hasattr(callback, "async_post_call_success_hook"):
                     await callback.async_post_call_success_hook(
                         user_api_key_dict=user_api_key_dict, response=new_response
@@ -461,25 +460,24 @@ async def post_call_success_hook(
 
     async def post_call_streaming_hook(
         self,
-        response: str,
+        response: AsyncIterator[Any],
         user_api_key_dict: UserAPIKeyAuth,
     ):
         """
         - Check outgoing streaming response uptil that point
         - Run through moderation check
         - Reject request if it fails moderation check
         """
-        new_response = copy.deepcopy(response)
-        snoop.pp(litellm.callbacks)
-        for callback in litellm.callbacks:
-            try:
-                if hasattr(callback, "async_post_call_streaming_hook"):
-                    await callback.async_post_call_streaming_hook(
-                        user_api_key_dict=user_api_key_dict, response=new_response
-                    )
-            except Exception as e:
-                raise e
-        return new_response
+        async for chunk in response:
+            for callback in litellm.callbacks:
+                try:
+                    if hasattr(callback, "async_post_call_streaming_hook"):
+                        await callback.async_post_call_streaming_hook(
+                            user_api_key_dict=user_api_key_dict, chunk=chunk,
+                        )
+                except Exception as e:
+                    raise e
+            yield chunk
 
 
 ### DB CONNECTOR ###