Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

AgentOps Integration #211

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

AgentOps Integration #211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 50 additions & 64 deletions gpt_computer_assistant/agent/process.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
import agentops
from dotenv import load_dotenv
import os

# Load environment variables
load_dotenv()

# Initialize AgentOps
agentops.init(os.getenv('AGENTOPS_API_KEY'))

try:
from ..llm import *
from .assistant import *
Expand All @@ -19,22 +29,16 @@
from utils.db import *
from utils.telemetry import my_tracer, os_name


import threading
import traceback


from pygame import mixer


import time

last_ai_response = None
user_id = load_user_id()
os_name_ = os_name()



@agentops.record_function('text_to_speech')
def tts_if_you_can(text:str, not_threaded=False, status_edit=False, bypass_other_settings = False):
try:
from ..gpt_computer_assistant import the_main_window
Expand All @@ -44,28 +48,26 @@ def tts_if_you_can(text:str, not_threaded=False, status_edit=False, bypass_other
signal_handler.assistant_response_ready.emit()

def play_audio():
for each_r in response_path:
mixer.init()
mixer.music.load(each_r)
mixer.music.play()
while mixer.music.get_busy():
if the_main_window.stop_talking:
mixer.music.stop()
break
time.sleep(0.1)
if status_edit:
signal_handler.assistant_response_stopped.emit()
for each_r in response_path:
mixer.init()
mixer.music.load(each_r)
mixer.music.play()
while mixer.music.get_busy():
if the_main_window.stop_talking:
mixer.music.stop()
break
time.sleep(0.1)
if status_edit:
signal_handler.assistant_response_stopped.emit()
if not not_threaded:
playback_thread = threading.Thread(target=play_audio)
playback_thread.start()
else:
play_audio()
except Exception as e:
pass



agentops.record_event('tts_error', {'error': str(e)})

@agentops.record_function('process_audio')
def process_audio(take_screenshot=True, take_system_audio=False, dont_save_image=False):
with my_tracer.start_span("process_audio") as span:
span.set_attribute("user_id", user_id)
Expand All @@ -75,35 +77,25 @@ def process_audio(take_screenshot=True, take_system_audio=False, dont_save_image
from ..gpt_computer_assistant import the_input_box, the_main_window
from ..audio.record import audio_data, the_input_box_pre


transcription = speech_to_text(mic_record_location)
agentops.record_event('transcription_completed', {'length': len(transcription)})

if take_system_audio:

transcription2 = speech_to_text(system_sound_location)
agentops.record_event('system_audio_transcription_completed', {'length': len(transcription2)})

llm_input = transcription

print("Previously AI response", last_ai_response, "end prev")

print("Input Box AI", the_input_box_pre)


if (
the_input_box_pre != ""
and not the_input_box_pre.startswith("System:")
and the_input_box_pre not in last_ai_response
):
if (the_input_box_pre != "" and not the_input_box_pre.startswith("System:") and the_input_box_pre not in last_ai_response):
llm_input += the_input_box_pre

if take_system_audio:
llm_input += " \n Other of USER: " + transcription2

if the_input_box.toPlainText().startswith("System:"):
the_main_window.update_from_thread("Transciption Completed. Running AI...")

the_main_window.update_from_thread("Transcription Completed. Running AI...")

print("LLM INPUT (screenshot)", llm_input)
agentops.record_event('llm_input_prepared', {'input_length': len(llm_input)})

llm_output = assistant(
llm_input,
Expand All @@ -112,17 +104,17 @@ def process_audio(take_screenshot=True, take_system_audio=False, dont_save_image
screenshot_path=screenshot_path if take_screenshot else None,
dont_save_image=dont_save_image,
)
agentops.record_event('llm_output_received', {'output_length': len(llm_output)})

if the_input_box.toPlainText().startswith("System:"):
the_main_window.update_from_thread("AI Response Completed. Generating Audio...")
last_ai_response = llm_output

from ..gpt_computer_assistant import the_main_window

signal_handler.assistant_response_ready.emit()

def play_text():
from ..gpt_computer_assistant import the_input_box, the_main_window

the_main_window.complated_answer = True
the_main_window.manuel_stop = True
while the_main_window.reading_thread or the_main_window.reading_thread_2:
Expand All @@ -141,35 +133,27 @@ def play_text():
the_main_window.update_from_thread("EXCEPTION: " + str(e))
tts_if_you_can("Exception occurred. Please check the logs.")
signal_handler.assistant_response_stopped.emit()
agentops.record_event('process_audio_error', {'error': str(e)})


@agentops.record_function('process_screenshot')
def process_screenshot():
with my_tracer.start_span("process_screenshot") as span:
span.set_attribute("user_id", user_id)
span.set_attribute("os_name", os_name_)
try:


global last_ai_response
from ..gpt_computer_assistant import the_input_box, the_main_window
from ..audio.record import audio_data, the_input_box_pre

llm_input = "I just take a screenshot. for you to remember. Just say 'Ok.' if the user doesnt want anything before."

llm_input = "I just take a screenshot. for you to remember. Just say 'Ok.' if the user doesnt want anything before."

if (
the_input_box_pre != ""
and not the_input_box_pre.startswith("System:")
and the_input_box_pre not in last_ai_response
):
if (the_input_box_pre != "" and not the_input_box_pre.startswith("System:") and the_input_box_pre not in last_ai_response):
llm_input += the_input_box_pre

print("LLM INPUT (just screenshot)", llm_input)
agentops.record_event('screenshot_llm_input_prepared', {'input_length': len(llm_input)})

if the_input_box.toPlainText().startswith("System:"):
the_main_window.update_from_thread("Transciption Completed. Running AI...")


the_main_window.update_from_thread("Transcription Completed. Running AI...")

llm_output = assistant(
llm_input,
Expand All @@ -178,17 +162,15 @@ def process_screenshot():
screenshot_path=just_screenshot_path,
dont_save_image=False,
)
agentops.record_event('screenshot_llm_output_received', {'output_length': len(llm_output)})

last_ai_response = llm_output

from ..gpt_computer_assistant import the_main_window

signal_handler.assistant_response_ready.emit()

def play_text():
from ..gpt_computer_assistant import the_input_box, the_main_window


the_main_window.complated_answer = True
the_main_window.manuel_stop = True
while the_main_window.reading_thread or the_main_window.reading_thread_2:
Expand All @@ -201,27 +183,25 @@ def play_text():
playback_thread = threading.Thread(target=play_text)
playback_thread.start()


except Exception as e:
print("Error in process_screenshot", e)
traceback.print_exc()
from ..gpt_computer_assistant import the_input_box, the_main_window
the_main_window.update_from_thread("EXCEPTION: " + str(e))
tts_if_you_can("Exception occurred. Please check the logs.")
signal_handler.assistant_response_stopped.emit()
agentops.record_event('process_screenshot_error', {'error': str(e)})



@agentops.record_function('process_text')
def process_text(text, screenshot_path=None):
with my_tracer.start_span("process_text") as span:
span.set_attribute("user_id", user_id)
span.set_attribute("os_name", os_name_)
try:

global last_ai_response

llm_input = text

agentops.record_event('text_llm_input_prepared', {'input_length': len(llm_input)})

llm_output = assistant(
llm_input,
Expand All @@ -230,6 +210,8 @@ def process_text(text, screenshot_path=None):
screenshot_path=screenshot_path,
dont_save_image=True,
)
agentops.record_event('text_llm_output_received', {'output_length': len(llm_output)})

last_ai_response = llm_output

from ..gpt_computer_assistant import the_main_window
Expand All @@ -242,8 +224,6 @@ def process_text(text, screenshot_path=None):

def play_text():
from ..gpt_computer_assistant import the_input_box, the_main_window


the_main_window.complated_answer = True
the_main_window.manuel_stop = True
while the_main_window.reading_thread or the_main_window.reading_thread_2:
Expand All @@ -256,13 +236,19 @@ def play_text():
playback_thread = threading.Thread(target=play_text)
playback_thread.start()


except Exception as e:
print("Error in process_text", e)
traceback.print_exc()
from ..gpt_computer_assistant import the_input_box, the_main_window
the_main_window.update_from_thread("EXCEPTION: " + str(e))
tts_if_you_can("Exception occurred. Please check the logs.")
signal_handler.assistant_response_stopped.emit()
agentops.record_event('process_text_error', {'error': str(e)})

# End the session when the program exits
import atexit

def end_session():
agentops.end_session('Success')

atexit.register(end_session)
5 changes: 5 additions & 0 deletions gpt_computer_assistant/gpt_computer_assistant.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from .audio.tts import text_to_speech
from .character import name, developer


except ImportError:
# This is for running the script directly
# in order to test the GUI without rebuilding the package
Expand All @@ -34,8 +35,10 @@
from utils.telemetry import my_tracer, os_name
from audio.wake_word import wake_word
from audio.tts import text_to_speech

import threading
import time
import agentops
import random
import math
from PyQt5.QtWidgets import QApplication, QMainWindow, QVBoxLayout, QWidget
Expand Down Expand Up @@ -84,6 +87,7 @@
user_id = load_user_id()
os_name_ = os_name()

agentops.init(os.getenv('AGENTOPS_API_KEY'))


readed_sentences = []
Expand Down Expand Up @@ -550,6 +554,7 @@ def mousePressEvent(self, event: QMouseEvent):

from PyQt5.QtCore import QVariantAnimation

@agentops.record_function('MainWindow')
class MainWindow(QMainWindow):
api_enabled = False
def __init__(self):
Expand Down