Journal_Labeling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/isaac-mackey/mind-uploading/blob/main/Journal_Labeling.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Install and configuration"
      ],
      "metadata": {
        "id": "ZO-HgDKvNayF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "# drive.mount('/content/drive', force_remount=True)\n",
        "\n",
        "!pip install openai\n",
        "from openai import OpenAI\n",
        "import json\n",
        "\n",
        "\n",
        "def show_json(obj):\n",
        "    display(json.loads(obj.model_dump_json()))"
      ],
      "metadata": {
        "id": "x6-L2du2ah7n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load entries and label-to-date dictionary"
      ],
      "metadata": {
        "id": "3DK9Nw-Z2chc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "def read_journal_entries(file_path):\n",
        "    with open(file_path, 'r') as file:\n",
        "        entries = []\n",
        "        current_entry = None\n",
        "\n",
        "        for line in file:\n",
        "            # Check if the line matches the pattern for a date entry\n",
        "            if re.match(r'^\\d{4}/\\d{2}/\\d{2}', line.strip()):\n",
        "                # If there's a previous entry, add it to the list\n",
        "                if current_entry:\n",
        "                    entries.append(current_entry)\n",
        "                # Start a new entry\n",
        "                current_entry = {'date': line.strip()}\n",
        "                current_entry['content'] = \"\"\n",
        "            elif current_entry:\n",
        "                # If it's not a date line and there's a current entry being processed, add the line to the content\n",
        "                current_entry['content'] += line\n",
        "\n",
        "        # Add the last entry after the loop\n",
        "        if current_entry:\n",
        "            entries.append(current_entry)\n",
        "\n",
        "    return entries\n",
        "\n",
        "# Replace 'file_path' with the path to your text file\n",
        "# file_path = '/content/drive/My Drive/march-2011-until-jan-2018-journal.txt'\n",
        "input_file_path = '/content/drive/My Drive/isaac_mackey@ucsb.edu 2023-11-13 17:45/Personal Journaling/full_journal_march_2011_to_march_2024.txt'\n",
        "\n",
        "entries = read_journal_entries(input_file_path)\n",
        "\n",
        "for year in [str(2011+x) for x in range(15)]:\n",
        "    num_entries_for_year = [e['date'][:4] for e in entries].count(year)\n",
        "    print(year+':',str(num_entries_for_year))\n",
        "\n",
        "# Print the dictionaries for each entry\n",
        "for entry in entries[:3]:\n",
        "    print(entry)\n",
        "    print(\"=\"*50)\n",
        "\n",
        "for entry in entries[-3:]:\n",
        "    print(entry)\n",
        "    print(\"=\"*50)\n",
        "\n",
        "entries_date_to_text = {}\n",
        "for item in entries:\n",
        "    yyyymmdd = item['date'][:10]\n",
        "    entry_text = item['content']\n",
        "    entries_date_to_text[yyyymmdd] = entry_text\n",
        "\n",
        "i = 0\n",
        "for k,v in entries_date_to_text.items():\n",
        "    if i == 2:\n",
        "        break\n",
        "    i += 1\n",
        "    print(k)\n",
        "    print(v)\n",
        "    print()\n",
        "\n",
        "with open('/content/drive/My Drive/isaac_mackey@ucsb.edu 2023-11-13 17:45/Personal Journaling/label-to-dates-json-gpt3-journal-1_2024_04_07_07:57:46.json', 'r') as f:\n",
        "  label_to_dates_dict = json.load(f)\n",
        "label_to_dates_dict = {x['label']:x['dates'] for x in label_to_dates_dict}\n",
        "label_to_dates_dict['Lost wallet']"
      ],
      "metadata": {
        "id": "VMtPnXfQaGU1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Upload files for retrieval"
      ],
      "metadata": {
        "id": "8Hxhhy25b5pA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "for x in os.listdir('/content/drive/My Drive/Personal Journaling'):\n",
        "  print(x)\n",
        "  if \"personal\" in x:\n",
        "    print(x)\n",
        "  if 'txt-' in x:\n",
        "    print(x)\n",
        "    # pass"
      ],
      "metadata": {
        "id": "o_6-WJ4i6RSn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_file_path = '/content/drive/My Drive/sms.json'\n",
        "\n",
        "# Replace with your actual file name\n",
        "with open(input_file_path, 'r') as file:\n",
        "    lines = file.readlines()\n",
        "\n",
        "output_file_path = input_file_path + '.jsonl'\n",
        "\n",
        "with open(output_file_path, 'w') as jsonl_file:\n",
        "    for line in lines:\n",
        "        # Assuming each line is a simple string\n",
        "        # Convert it into a JSON object with a specific key\n",
        "        json_obj = json.dumps({\"data\": line.strip()})\n",
        "        jsonl_file.write(json_obj + '\\n')"
      ],
      "metadata": {
        "id": "AiZEJjSKGItD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# For Fine-Tune\n",
        "# data_file = '/content/drive/My Drive/sms-20231224020008.xml-role-system-user-5.json.jsonl'\n",
        "\n",
        "# For Assistants (Retrieval)\n",
        "data_file = '/content/drive/My Drive/Personal Journaling/full_journal.txt'\n",
        "\n",
        "response = client.files.create(\n",
        "  file=open(data_file, \"rb\"),\n",
        "  # purpose=\"fine-tune\"\n",
        "  purpose=\"assistants\"\n",
        ")"
      ],
      "metadata": {
        "id": "WfaUJ3AMj5ds"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "show_json(response)"
      ],
      "metadata": {
        "id": "sJLHxcnH8F-7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load journal entries"
      ],
      "metadata": {
        "id": "BqrR10-x-Vu2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "def read_journal_entries(file_path):\n",
        "    with open(file_path, 'r') as file:\n",
        "        entries = []\n",
        "        current_entry = None\n",
        "\n",
        "        for line in file:\n",
        "            # Check if the line matches the pattern for a date entry\n",
        "            if re.match(r'^\\d{4}/\\d{2}/\\d{2}', line.strip()):\n",
        "                # If there's a previous entry, add it to the list\n",
        "                if current_entry:\n",
        "                    entries.append(current_entry)\n",
        "                # Start a new entry\n",
        "                current_entry = {'date': line.strip()}\n",
        "                current_entry['content'] = \"\"\n",
        "            elif current_entry:\n",
        "                # If it's not a date line and there's a current entry being processed, add the line to the content\n",
        "                current_entry['content'] += line\n",
        "\n",
        "        # Add the last entry after the loop\n",
        "        if current_entry:\n",
        "            entries.append(current_entry)\n",
        "\n",
        "    return entries\n",
        "\n",
        "# Replace 'file_path' with the path to your text file\n",
        "# file_path = '/content/drive/My Drive/march-2011-until-jan-2018-journal.txt'\n",
        "input_file_path = '/content/drive/My Drive/isaac_mackey@ucsb.edu 2023-11-13 17:45/Personal Journaling/full_journal_march_2011_to_march_2024.txt'\n",
        "\n",
        "entries = read_journal_entries(input_file_path)\n",
        "\n",
        "for year in [str(2011+x) for x in range(15)]:\n",
        "    num_entries_for_year = [e['date'][:4] for e in entries].count(year)\n",
        "    print(year+':',str(num_entries_for_year))\n",
        "\n",
        "# Print the dictionaries for each entry\n",
        "for entry in entries[:3]:\n",
        "    print(entry)\n",
        "    print(\"=\"*50)\n",
        "\n",
        "for entry in entries[-3:]:\n",
        "    print(entry)\n",
        "    print(\"=\"*50)"
      ],
      "metadata": {
        "id": "191QZzKj-Ur9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "entries_date_to_text = {}\n",
        "for item in entries:\n",
        "    yyyymmdd = item['date'][:10]\n",
        "    entry_text = item['content']\n",
        "    entries_date_to_text[yyyymmdd] = entry_text"
      ],
      "metadata": {
        "id": "h5vPZS7Ndheu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "i = 0\n",
        "for k,v in entries_date_to_text.items():\n",
        "    if i == 2:\n",
        "        break\n",
        "    i += 1\n",
        "    print(k)\n",
        "    print(v)\n",
        "    print()"
      ],
      "metadata": {
        "id": "TgB80UL-iDh_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Journal entry topic labeling with function tools"
      ],
      "metadata": {
        "id": "86DL2IP5w4Cv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "topics = ['childhood', 'family', 'academics', 'education', 'goals', 'health', 'exercise', 'hobbies', 'travel', 'friends', 'morals', 'career', 'relationships', 'technology', 'art', 'culture', 'finance', 'environment', 'sports', 'food', 'entertainment']\n",
        "\n",
        "longer_topics = [\n",
        "    'childhood', 'family', 'academics', 'education', 'goals', 'health', 'exercise', 'hobbies', 'travel',\n",
        "    'friends', 'morals', 'career', 'relationships', 'technology', 'art', 'culture', 'finance', 'environment',\n",
        "    'sports', 'food', 'entertainment', 'routines',\n",
        "    'passion', 'emotions', 'intelligence', 'core values', 'identity', 'challenges', 'achievements',\n",
        "    'inspirations', 'reflections', 'ambitions', 'social',\n",
        "    'communication', 'humor', 'culture',\n",
        "    'conflicts', 'comfort', 'community', 'tradition',\n",
        "    'language', 'leadership', 'decisions', 'future',\n",
        "    'social media', 'sleep', 'reading', 'music'\n",
        "]\n",
        "\n",
        "system_message = \"Tell me information like keywords of relevance about some diary entries.\"\n",
        "\n",
        "my_assistant = client.beta.assistants.create(\n",
        "    instructions=system_message,\n",
        "    name=\"Diary Labeling Assistant\",\n",
        "    tools=[#{\"type\": \"retrieval\"},\n",
        "           {\n",
        "      \"type\": \"function\",\n",
        "      \"function\": {\n",
        "          \"name\": \"getRelevantTopics\",\n",
        "          \"description\": \"Return the topics are relevant to a diary entry\",\n",
        "          \"parameters\": {\n",
        "              \"type\": \"object\",\n",
        "              \"properties\": {\n",
        "                  \"entry\": {\"type\": \"string\"},\n",
        "                  \"keywords\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Keywords to determine relevance\"}\n",
        "                            },\n",
        "              \"required\": [\"entry\", \"keywords\"]\n",
        "                        }\n",
        "                  }\n",
        "          },\n",
        "           ],\n",
        "    model='gpt-3.5-turbo-1106',\n",
        "    #model=\"gpt-4-turbo-preview\",\n",
        "    file_ids= [] #[files_to_retrieve]\n",
        ")"
      ],
      "metadata": {
        "id": "LU7FwMEA4FX8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Example function calling"
      ],
      "metadata": {
        "id": "BXU_miLnFv7j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from openai import OpenAI\n",
        "import json\n",
        "\n",
        "client = OpenAI(\n",
        ")\n",
        "\n",
        "# Example dummy function hard coded to return the same weather\n",
        "# In production, this could be your backend API or an external API\n",
        "def get_current_weather(location, unit=\"fahrenheit\"):\n",
        "    \"\"\"Get the current weather in a given location\"\"\"\n",
        "    if \"tokyo\" in location.lower():\n",
        "        return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": unit})\n",
        "    elif \"san francisco\" in location.lower():\n",
        "        return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": unit})\n",
        "    elif \"paris\" in location.lower():\n",
        "        return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": unit})\n",
        "    else:\n",
        "        return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n",
        "\n",
        "def run_conversation():\n",
        "    # Step 1: send the conversation and available functions to the model\n",
        "    messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n",
        "    tools = [\n",
        "        {\n",
        "            \"type\": \"function\",\n",
        "            \"function\": {\n",
        "                \"name\": \"get_current_weather\",\n",
        "                \"description\": \"Get the current weather in a given location\",\n",
        "                \"parameters\": {\n",
        "                    \"type\": \"object\",\n",
        "                    \"properties\": {\n",
        "                        \"location\": {\n",
        "                            \"type\": \"string\",\n",
        "                            \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
        "                        },\n",
        "                        \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
        "                    },\n",
        "                    \"required\": [\"location\"],\n",
        "                },\n",
        "            },\n",
        "        }\n",
        "    ]\n",
        "    response = client.chat.completions.create(\n",
        "        model=\"gpt-3.5-turbo-0125\",\n",
        "        messages=messages,\n",
        "        tools=tools,\n",
        "        tool_choice=\"auto\",  # auto is default, but we'll be explicit\n",
        "    )\n",
        "    response_message = response.choices[0].message\n",
        "    tool_calls = response_message.tool_calls\n",
        "    # Step 2: check if the model wanted to call a function\n",
        "    if tool_calls:\n",
        "        # Step 3: call the function\n",
        "        # Note: the JSON response may not always be valid; be sure to handle errors\n",
        "        available_functions = {\n",
        "            \"get_current_weather\": get_current_weather,\n",
        "        }  # only one function in this example, but you can have multiple\n",
        "        messages.append(response_message)  # extend conversation with assistant's reply\n",
        "        # Step 4: send the info for each function call and function response to the model\n",
        "        for tool_call in tool_calls:\n",
        "            function_name = tool_call.function.name\n",
        "            function_to_call = available_functions[function_name]\n",
        "            function_args = json.loads(tool_call.function.arguments)\n",
        "            function_response = function_to_call(\n",
        "                location=function_args.get(\"location\"),\n",
        "                unit=function_args.get(\"unit\"),\n",
        "            )\n",
        "            messages.append(\n",
        "                {\n",
        "                    \"tool_call_id\": tool_call.id,\n",
        "                    \"role\": \"tool\",\n",
        "                    \"name\": function_name,\n",
        "                    \"content\": function_response,\n",
        "                }\n",
        "            )  # extend conversation with function response\n",
        "        second_response = client.chat.completions.create(\n",
        "            model=\"gpt-3.5-turbo-0125\",\n",
        "            messages=messages,\n",
        "        )  # get a new response from the model where it can see the function response\n",
        "        return second_response\n",
        "print(run_conversation())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_mOWFh9R9IHZ",
        "outputId": "025cf84c-102b-433b-83b4-b15977055f4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ChatCompletion(id='chatcmpl-96fMIZV2SrkQsQUh8qDuiP7QLSGCB', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='The current weather in:\\n\\n- San Francisco: 72°C\\n- Tokyo: 10°C\\n- Paris: 22°C', role='assistant', function_call=None, tool_calls=None))], created=1711375978, model='gpt-3.5-turbo-0125', object='chat.completion', system_fingerprint='fp_3bc1b5746c', usage=CompletionUsage(completion_tokens=26, prompt_tokens=169, total_tokens=195))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Actual label function calling"
      ],
      "metadata": {
        "id": "XjrXEvjuP4Am"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "import random\n",
        "\n",
        "def wait_on_run(run, thread):\n",
        "    print(\"wait on run\")\n",
        "    # show_json(run)\n",
        "    while run.status == \"queued\" or run.status == \"in_progress\":\n",
        "        run = client.beta.threads.runs.retrieve(\n",
        "            thread_id=thread.id,\n",
        "            run_id=run.id,\n",
        "        )\n",
        "        time.sleep(0.5)\n",
        "    return run\n",
        "\n",
        "def wait_on_run_action(run, thread):\n",
        "    print(\"wait on run action\")\n",
        "    show_json(run)\n",
        "    while run.status == \"queued\" or run.status == \"in_progress\": # or run.status == \"requires_action\":\n",
        "        run = client.beta.threads.runs.retrieve(\n",
        "            thread_id=thread.id,\n",
        "            run_id=run.id,\n",
        "        )\n",
        "        print(\"wait on run action loop\")\n",
        "        show_json(run)\n",
        "        time.sleep(0.5)\n",
        "    return run\n",
        "\n",
        "def callTools(tool_calls):\n",
        "# the parameter comes straight from the openAI run\n",
        "    tool_outputs = []\n",
        "    for t in tool_calls:\n",
        "        functionName = t.function.name\n",
        "        attributes = json.loads(t.function.arguments)\n",
        "        try:\n",
        "            functionResponse =globals()[functionName](attributes)\n",
        "        except:\n",
        "             # we just tell openAi we couldn't :)\n",
        "            functionResponse = { \"status\" : 'Error in function call '+functionName+'('+t.function.arguments+')' }\n",
        "        tool_outputs.append(  { \"tool_call_id\": t.id , \"output\": json.dumps(functionResponse) })\n",
        "    return tool_outputs\n",
        "\n",
        "entry_labels = []\n",
        "\n",
        "i = 0\n",
        "\n",
        "for entry in entries:\n",
        "    print('i:',str(i),'out of',str(len(entries)))\n",
        "    i += 1\n",
        "    date = entry['date']\n",
        "    content = entry['content']\n",
        "\n",
        "    thread = client.beta.threads.create()\n",
        "    # show_json(thread)\n",
        "\n",
        "    content = \"From the following topics: \"+str(longer_topics)+\", list the topics related to the following entry: \"+content\n",
        "\n",
        "    # print(content)\n",
        "\n",
        "    message = client.beta.threads.messages.create(\n",
        "        thread_id=thread.id,\n",
        "        role=\"user\",\n",
        "        content=content,\n",
        "        file_ids=[] #files_to_retrieve\n",
        "    )\n",
        "    # show_json(message)\n",
        "\n",
        "    run = client.beta.threads.runs.create(\n",
        "    thread_id=thread.id,\n",
        "    assistant_id=my_assistant.id\n",
        "    )\n",
        "\n",
        "    run = wait_on_run(run, thread)\n",
        "\n",
        "    if run.status == 'completed' or run.status == 'requires_action':\n",
        "      messages = client.beta.threads.messages.list(\n",
        "        thread_id=thread.id\n",
        "      )\n",
        "      # show_json(messages)\n",
        "      m = []\n",
        "      for y in messages:\n",
        "          # print(y)\n",
        "          m.append(y)\n",
        "      m.reverse()\n",
        "\n",
        "      # print('keywords')\n",
        "      keywords = json.loads(run.required_action.submit_tool_outputs.tool_calls[0].function.arguments)['keywords']\n",
        "      # print(keywords)\n",
        "\n",
        "      # print(m)\n",
        "      # print(m[0])\n",
        "\n",
        "      answer = \"\"\n",
        "      for x in m:\n",
        "          if x.assistant_id:\n",
        "              answer += ' '+x.content[0].text.value\n",
        "      entry_labels.append((date, keywords))\n",
        "    else:\n",
        "      print(\"run.status\")\n",
        "      print(run.status)"
      ],
      "metadata": {
        "id": "BRHrQ6Qdw7iD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for x in entry_labels:\n",
        "    print(x[0])\n",
        "    print(str(x[1]))"
      ],
      "metadata": {
        "id": "Hwg5cj69KSrw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "json.loads(action_run.required_action.submit_tool_outputs.tool_calls[0].function.arguments)['keywords']"
      ],
      "metadata": {
        "id": "Y6T2PAVjTTtG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "functionName = t.function.name\n",
        "attributes = json.loads(t.function.arguments)\n",
        "try:\n",
        "    functionResponse =globals()[functionName](attributes)\n",
        "except:\n",
        "      # we just tell openAi we couldn't :)\n",
        "    functionResponse = { \"status\" : 'Error in function call '+functionName+'('+t.function.arguments+')' }\n",
        "tool_outputs.append(  { \"tool_call_id\": t.id , \"output\": json.dumps(functionResponse) })"
      ],
      "metadata": {
        "id": "b6IAxhkFWBo-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Journal entry topic labeling without function tools"
      ],
      "metadata": {
        "id": "jGg5dUaNOyl8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "topics = ['childhood', 'family', 'academics', 'education', 'goals', 'health', 'exercise', 'hobbies', 'travel', 'friends', 'morals', 'career', 'relationships', 'technology', 'art', 'culture', 'finance', 'environment', 'sports', 'food', 'entertainment',\n",
        "          'military', 'swimming', 'cycling', 'wrestling', 'physics']\n",
        "\n",
        "longer_topics = [\n",
        "    'childhood', 'family', 'academics', 'education', 'goals', 'health', 'exercise', 'hobbies', 'travel',\n",
        "    'friends', 'morals', 'career', 'relationships', 'technology', 'art', 'culture', 'finance', 'environment',\n",
        "    'sports', 'food', 'entertainment', 'routines',\n",
        "    'passion', 'emotions', 'intelligence', 'core values', 'identity', 'challenges', 'achievements',\n",
        "    'inspirations', 'reflections', 'ambitions', 'social',\n",
        "    'communication', 'humor', 'culture',\n",
        "    'conflicts', 'comfort', 'community', 'tradition',\n",
        "    'language', 'leadership', 'decisions', 'future',\n",
        "    'social media', 'sleep', 'reading', 'music'\n",
        "]\n",
        "\n",
        "pure_system_message = \"You are an assistant that identifies and extracts keywords of relevance from a person's diary entries.\"\n",
        "\n",
        "my_assistant = client.beta.assistants.create(\n",
        "    instructions=pure_system_message,\n",
        "    name=\"Diary Entry Labeling Assistant\",\n",
        "    tools=[],\n",
        "    model='gpt-3.5-turbo-1106',\n",
        "    #model=\"gpt-4-turbo-preview\",\n",
        "    file_ids= [] #[files_to_retrieve]\n",
        ")"
      ],
      "metadata": {
        "id": "dEGHSLRkOwGz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "import random\n",
        "\n",
        "def wait_on_run(run, thread):\n",
        "    print(\"wait on run\")\n",
        "    # show_json(run)\n",
        "    while run.status == \"queued\" or run.status == \"in_progress\":\n",
        "        run = client.beta.threads.runs.retrieve(\n",
        "            thread_id=thread.id,\n",
        "            run_id=run.id,\n",
        "        )\n",
        "        time.sleep(0.5)\n",
        "    return run\n",
        "\n",
        "created_entry_labels = []\n",
        "\n",
        "i = 0\n",
        "\n",
        "for entry in entries:\n",
        "    print('i:',str(i),'out of',str(len(entries)))\n",
        "    i += 1\n",
        "    date = entry['date']\n",
        "    content = entry['content']\n",
        "\n",
        "    thread = client.beta.threads.create()\n",
        "    # show_json(thread)\n",
        "\n",
        "    content = \"Return a Python list of keywords that could be used to identify or classify the following entry: \"+content\n",
        "\n",
        "    # print(content)\n",
        "\n",
        "    message = client.beta.threads.messages.create(\n",
        "        thread_id=thread.id,\n",
        "        role=\"user\",\n",
        "        content=content,\n",
        "        file_ids=[] #files_to_retrieve\n",
        "    )\n",
        "    # show_json(message)\n",
        "\n",
        "    run = client.beta.threads.runs.create(\n",
        "    thread_id=thread.id,\n",
        "    assistant_id=my_assistant.id\n",
        "    )\n",
        "\n",
        "    run = wait_on_run(run, thread)\n",
        "\n",
        "    if run.status == 'completed' or run.status == 'requires_action':\n",
        "      messages = client.beta.threads.messages.list(\n",
        "        thread_id=thread.id\n",
        "      )\n",
        "      # show_json(messages)\n",
        "      m = []\n",
        "      for y in messages:\n",
        "          # print(y)\n",
        "          m.append(y)\n",
        "      m.reverse()\n",
        "\n",
        "      answer = \"\"\n",
        "      for x in m:\n",
        "          if x.assistant_id:\n",
        "              answer += ' '+x.content[0].text.value\n",
        "      created_entry_labels.append((date, answer))\n",
        "    else:\n",
        "      print(\"run.status\")\n",
        "      print(run.status)"
      ],
      "metadata": {
        "id": "hvv7GkUCO9aV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for x in created_entry_labels[100:104]:\n",
        "  print(x[0])\n",
        "  print(x[1])\n",
        "  print()"
      ],
      "metadata": {
        "id": "i08MgNKpF5Xs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "def extract_keywords_numbered(diary_entry):\n",
        "    # Use regular expression to find the lines containing keywords\n",
        "    keyword_lines = re.findall(r'\\d+\\.\\s+(.*)', diary_entry)\n",
        "\n",
        "    # Extract keywords from the lines\n",
        "    keywords = [line.strip() for line in keyword_lines]\n",
        "\n",
        "    return keywords\n",
        "\n",
        "def extract_keywords_dashed(diary_entry):\n",
        "    # Use regular expression to find the lines containing keywords\n",
        "    keyword_lines = re.findall(r'-\\s*(.*)', diary_entry)\n",
        "\n",
        "    # Extract keywords from the lines\n",
        "    keywords = [line.strip() for line in keyword_lines]\n",
        "\n",
        "    return keywords\n",
        "\n",
        "# Example usage\n",
        "diary_entry = \"\"\"2011/07/01 (Friday)\n",
        "Based on the provided diary entry, the following keywords could be used to identify or classify the entry:\n",
        "\n",
        "1. Cycling\n",
        "2. Adventure\n",
        "3. Railroad tracks\n",
        "4. Mile-long straight aways\n",
        "5. Bonding\n",
        "6. Gas station\n",
        "7. Parenting\n",
        "8. Rules\n",
        "9. Mistakes\n",
        "10. Disappearance\n",
        "11. Responsibility\n",
        "12. Death\"\"\"\n",
        "\n",
        "keywords = extract_keywords_numbered(diary_entry)\n",
        "print(keywords)"
      ],
      "metadata": {
        "id": "bct5YXO5Y-hg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "created_entry_label_dict = {}\n",
        "i = 0\n",
        "j = 0\n",
        "jj = 0\n",
        "k = 0\n",
        "for x,y in created_entry_labels:\n",
        "    if x not in created_entry_label_dict:\n",
        "        try:\n",
        "            my_list = eval(y)\n",
        "            if type(my_list) == type([]):\n",
        "                created_entry_label_dict[x] = my_list\n",
        "                i += 1\n",
        "        except SyntaxError:\n",
        "            y = y.replace(\"```python\", \"\").replace(\"```\", \"\").strip()\n",
        "            try:\n",
        "                my_list = eval(y)\n",
        "                if type(my_list) == type([]):\n",
        "                    created_entry_label_dict[x] = my_list\n",
        "                    j += 1\n",
        "            except Exception as e:\n",
        "                jj += 1\n",
        "        except Exception as e:\n",
        "            k += 1\n",
        "print(\"i\",str(i))\n",
        "print(\"j\",str(j))\n",
        "print(\"jj\",str(jj))\n",
        "print(\"k\",str(k))"
      ],
      "metadata": {
        "id": "D5dgZe6R8hAM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "created_entry_label_dict = {}\n",
        "i = 0\n",
        "ii = 0\n",
        "j = 0\n",
        "jj = 0\n",
        "k = 0\n",
        "for x,y in created_entry_labels:\n",
        "    if x not in created_entry_label_dict:\n",
        "        try:\n",
        "            my_list = extract_keywords_numbered(y)\n",
        "            if type(my_list) == type([]) and len(my_list) > 0:\n",
        "                created_entry_label_dict[x] = my_list\n",
        "                i += 1\n",
        "            else:\n",
        "                my_list = extract_keywords_dashed(y)\n",
        "                if type(my_list) == type([]) and len(my_list) > 0:\n",
        "                    created_entry_label_dict[x] = my_list\n",
        "                    ii += 1\n",
        "        except SyntaxError:\n",
        "            y = y.replace(\"```python\", \"\").replace(\"```\", \"\").strip()\n",
        "            try:\n",
        "                my_list = eval(y)\n",
        "                if type(my_list) == type([]):\n",
        "                    created_entry_label_dict[x] = my_list\n",
        "                    j += 1\n",
        "            except Exception as e:\n",
        "                jj += 1\n",
        "        except Exception as e:\n",
        "            k += 1\n",
        "print(\"i\",str(i))\n",
        "print(\"ii\",str(ii))\n",
        "print(\"j\",str(j))\n",
        "print(\"jj\",str(jj))\n",
        "print(\"k\",str(k))"
      ],
      "metadata": {
        "id": "GvfVHhMzZXIq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "i = 0\n",
        "for x,y in created_entry_label_dict.items():\n",
        "    print(x)\n",
        "    print(y)\n",
        "    print()\n",
        "    i += 1\n",
        "    if i == 20:\n",
        "        break"
      ],
      "metadata": {
        "id": "Ap62yRVsPCqh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "\n",
        "# Sample dictionary a and list b\n",
        "a = created_entry_label_dict\n",
        "\n",
        "# Flatten the values in dictionary a into a single list\n",
        "all_labels = [item for sublist in a.values() for item in sublist]\n",
        "\n",
        "# Count occurrences of each value in b within all_values\n",
        "counts = Counter(all_labels)\n",
        "\n",
        "all_labels = list(set(all_labels))\n",
        "\n",
        "label_to_date_dict = {item: [] for item in all_labels}\n",
        "\n",
        "for date,label_list in created_entry_label_dict.items():\n",
        "    for label in label_list:\n",
        "        # Append the key to the list of keys for the corresponding value in the mapping dictionary\n",
        "        label_to_date_dict[label].append(date)\n",
        "\n",
        "i = 0\n",
        "for label,dates in label_to_date_dict.items():\n",
        "    print(label)\n",
        "    print(dates)\n",
        "    print()\n",
        "    i += 1\n",
        "    if i == 5:\n",
        "        break\n",
        "\n",
        "# Output the counts\n",
        "for item in all_labels:\n",
        "    if counts[item] > 10:\n",
        "        print(f\"{item}: {counts[item]}\")"
      ],
      "metadata": {
        "id": "3rEZiA1p_eBY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Save Label-to-Dates-Dict"
      ],
      "metadata": {
        "id": "4xgDhMXDOuf1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "label_to_dates_json_list = []\n",
        "\n",
        "for label,dates in label_to_date_dict.items():\n",
        "    label_to_dates_json_list.append({\"label\":label,\"dates\":dates})\n",
        "\n",
        "json_string = json.dumps(label_to_dates_json_list)\n",
        "\n",
        "conversation_name = \"label-to-dates-json-gpt3-journal-1\"\n",
        "\n",
        "print(\"Conversation name:\",conversation_name)\n",
        "\n",
        "conversation_name_check = input(\"Confirm conversation name (Y) or provide a name for the conversation: \")\n",
        "# Validate the conversation name\n",
        "\n",
        "if not conversation_name_check.lower() == \"y\":\n",
        "    conversation_name = conversation_name_check\n",
        "\n",
        "while \" \" in conversation_name:\n",
        "  print(\"Invalid conversation name. Please enter a name without spaces.\")\n",
        "  conversation_name = input(\"Provide a name for the conversation: \")\n",
        "\n",
        "import datetime\n",
        "eastern_now_str = datetime.datetime.utcnow().astimezone(datetime.timezone(datetime.timedelta(hours=-4))).strftime(\"%Y_%m_%d_%H:%M:%S\")\n",
        "conversation_date = eastern_now_str\n",
        "\n",
        "# Create the file name.\n",
        "file_name = f\"{conversation_name}_{conversation_date}\"\n",
        "\n",
        "print('json file_name',file_name)\n",
        "\n",
        "# Save the JSON string to a file in Google Drive.\n",
        "with open(f\"/content/drive/My Drive/isaac_mackey@ucsb.edu 2023-11-13 17:45/Personal Journaling//{file_name}.json\", \"w\") as f:\n",
        "    f.write(json_string)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wzUr073cXtWu",
        "outputId": "1afd3d20-976c-43e8-9429-bee6fa3ec703"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Conversation name: label-to-dates-json-gpt3-journal-1\n",
            "Confirm conversation name (Y) or provide a name for the conversation: y\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Save date-to-labels dict"
      ],
      "metadata": {
        "id": "bj6Sf_OSUXfA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "conversation_name = \"date-to-labels-json-gpt3-journal-1\"\n",
        "\n",
        "print(\"Conversation name:\",conversation_name)\n",
        "\n",
        "conversation_name_check = input(\"Confirm conversation name (Y) or provide a name for the conversation: \")\n",
        "# Validate the conversation name\n",
        "\n",
        "if not conversation_name_check.lower() == \"y\":\n",
        "    conversation_name = conversation_name_check\n",
        "\n",
        "while \" \" in conversation_name:\n",
        "  print(\"Invalid conversation name. Please enter a name without spaces.\")\n",
        "  conversation_name = input(\"Provide a name for the conversation: \")\n",
        "\n",
        "import datetime\n",
        "eastern_now_str = datetime.datetime.utcnow().astimezone(datetime.timezone(datetime.timedelta(hours=-4))).strftime(\"%Y_%m_%d_%H:%M:%S\")\n",
        "conversation_date = eastern_now_str\n",
        "\n",
        "# Create the file name.\n",
        "file_name = f\"{conversation_name}_{conversation_date}\"\n",
        "\n",
        "print('json file_name',file_name)\n",
        "\n",
        "with open(f'/content/drive/My Drive/isaac_mackey@ucsb.edu 2023-11-13 17:45/Personal Journaling//{file_name}.json', 'w') as f:\n",
        "  json.dump(created_entry_label_dict, f)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZHxp43HvWvU7",
        "outputId": "4f7ca4bb-373e-4091-8a67-cebad73ea370"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Conversation name: date-to-labels-json-gpt3-journal-1\n",
            "Confirm conversation name (Y) or provide a name for the conversation: y\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load label_to_dates dict"
      ],
      "metadata": {
        "id": "1fR64VYZZKQf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open('/content/drive/My Drive/isaac_mackey@ucsb.edu 2023-11-13 17:45/Personal Journaling/label-to-dates-json-gpt3-journal-1_2024_04_07_07:57:46.json', 'r') as f:\n",
        "  label_to_dates_dict = json.load(f)\n",
        "label_to_dates_dict = {x['label']:x['dates'] for x in label_to_dates_dict}\n",
        "label_to_dates_dict['Lost wallet']"
      ],
      "metadata": {
        "id": "UbCQKfXKZMpk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Print and search labels"
      ],
      "metadata": {
        "id": "_xKa9L4NZpIB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# prompt: print the top k labels with the most dates in their entry in label_to_date_dict\n",
        "\n",
        "# First, we need to count the number of dates associated with each label.\n",
        "label_counts = {}\n",
        "for label, dates in label_to_dates_dict.items():\n",
        "    label_counts[label] = len(dates)\n",
        "\n",
        "# Then, we can sort the labels by their counts in descending order.\n",
        "sorted_labels = sorted(label_counts, key=label_counts.get, reverse=True)\n",
        "\n",
        "# Finally, we can print the top k labels with the most dates.\n",
        "k = 20  # Change this value to adjust the number of labels to print.\n",
        "for label in sorted_labels[:k]:\n",
        "    print(f\"Label: {label}, Dates: {label_counts[label]}\")"
      ],
      "metadata": {
        "id": "bu1bQF1tTuL7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# prompt: Write a user input query to get a few keyword successively and print the text of the journal entris that are labeled with any of those keywords\n",
        "\n",
        "import random\n",
        "\n",
        "keyword = \"start\"\n",
        "keyword_list = []\n",
        "\n",
        "top_k = 10\n",
        "random_k = 10\n",
        "\n",
        "for label in random.sample(sorted_labels[:top_k*3], top_k) + random.sample(sorted_labels, random_k):\n",
        "    print(f\"Label: {label}, Dates: {label_counts[label]}\")\n",
        "\n",
        "while keyword != \"\":\n",
        "    print(\"\\nEnter a keyword to get the related journal entries: (enter blank to stop)\")\n",
        "    keyword = input()\n",
        "    try:\n",
        "        print(\"You entered: \", keyword,',',str(len(label_to_dates_dict[keyword])),'entries')\n",
        "    except KeyError:\n",
        "        print(\"You entered: \", keyword,', 0 entries')\n",
        "    keyword_list.append(keyword)\n",
        "\n",
        "dates_selected = []\n",
        "\n",
        "for keyword in keyword_list:\n",
        "    if keyword in label_to_dates_dict:\n",
        "        for date in label_to_dates_dict[keyword]:\n",
        "            dates_selected.append(date[:10])\n",
        "\n",
        "dates_selected = list(set(dates_selected))\n",
        "\n",
        "window_size = 80\n",
        "\n",
        "if not dates_selected:\n",
        "    print(\"No entries found for \"+' '.join(keyword_list))\n",
        "else:\n",
        "    for date in list(set(dates_selected)):\n",
        "        print(\"Date:\", date)\n",
        "        print()\n",
        "        entry = entries_date_to_text[date]\n",
        "        lines = entry.split(\"\\n\")\n",
        "        for line in lines:\n",
        "          if line == \"\":\n",
        "            # print()  # Print a blank line\n",
        "            continue\n",
        "          for i in range(0, len(line), window_size):\n",
        "            print(line[i:i + window_size])\n",
        "          print()  # Print a newline after each block\n",
        "        print('#'*window_size)\n",
        "        print()"
      ],
      "metadata": {
        "id": "VpJJenacctxf"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}