001_haystack_combio_first_experiments-redacted.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "908657c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "logging.basicConfig(format=\"%(levelname)s - %(name)s -  %(message)s\", level=logging.WARNING)\n",
    "logging.getLogger(\"haystack\").setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e59765eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recommended: Start Elasticsearch using Docker via the Haystack utility function\n",
    "from haystack.utils import launch_es\n",
    "\n",
    "launch_es()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "99f4f22b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "time.sleep(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "29652b83",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from haystack.document_stores import ElasticsearchDocumentStore\n",
    "\n",
    "# Get the host where Elasticsearch is running, default to localhost\n",
    "host = os.environ.get(\"ELASTICSEARCH_HOST\", \"localhost\")\n",
    "document_store = ElasticsearchDocumentStore(host=host, username=\"\", password=\"\", index=\"document\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "300dd7c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from haystack.utils import clean_wiki_text, convert_files_to_docs, fetch_archive_from_http\n",
    "\n",
    "\n",
    "# Let's first fetch some documents that we want to query\n",
    "# Here: 517 Wikipedia articles for Game of Thrones\n",
    "doc_dir = \"/Users/kpham/Desktop/coding-kim/haystack/combio_clean/singletxt\"\n",
    "# s3_url = \"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt1.zip\"\n",
    "# fetch_archive_from_http(url=s3_url, output_dir=doc_dir)\n",
    "\n",
    "# Convert files to dicts\n",
    "# You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers)\n",
    "# It must take a str as input, and return a str.\n",
    "docs = convert_files_to_docs(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True)\n",
    "\n",
    "# We now have a list of dictionaries that we can write to our document store.\n",
    "# If your texts come from a different source (e.g. a DB), you can of course skip convert_files_to_dicts() and create the dictionaries yourself.\n",
    "# The default format here is:\n",
    "# {\n",
    "#    'content': \"<DOCUMENT_TEXT_HERE>\",\n",
    "#    'meta': {'name': \"<DOCUMENT_NAME_HERE>\", ...}\n",
    "# }\n",
    "# (Optionally: you can also add more key-value-pairs here, that will be indexed as fields in Elasticsearch and\n",
    "# can be accessed later for filtering or shown in the responses of the Pipeline)\n",
    "\n",
    "# Let's have a look at the first 3 entries:\n",
    "print(docs[:3])\n",
    "\n",
    "# Now, let's write the dicts containing documents to our DB.\n",
    "document_store.write_documents(docs)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f594dcba",
   "metadata": {},
   "outputs": [],
   "source": [
    "from haystack.nodes import BM25Retriever\n",
    "\n",
    "retriever = BM25Retriever(document_store=document_store)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a83e32be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dba73f61abf49bd82a6f7c821f1ad1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading config.json:   0%|          | 0.00/571 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30bc7afc16ef4d388c0f4c9ab6db2f33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/473M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d93e58a68a974a8ba2519e14a392320a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer_config.json:   0%|          | 0.00/79.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e40bc37a609f4220b0063da4783220c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading vocab.json:   0%|          | 0.00/878k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ed730d5d3da420bbe5df59091425e20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading merges.txt:   0%|          | 0.00/446k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c16c01499fc54633b0ccf361b4b078e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading special_tokens_map.json:   0%|          | 0.00/772 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from haystack.nodes import FARMReader\n",
    "\n",
    "# Load a  local model or any of the QA models on\n",
    "# Hugging Face's model hub (https://huggingface.co/models)\n",
    "\n",
    "reader = FARMReader(model_name_or_path=\"deepset/roberta-base-squad2\", use_gpu=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2a4f90fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from haystack.pipelines import ExtractiveQAPipeline\n",
    "\n",
    "pipe = ExtractiveQAPipeline(reader, retriever)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b8d1fda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can configure how many candidates the Reader and Retriever shall return\n",
    "# The higher top_k_retriever, the better (but also the slower) your answers.\n",
    "prediction = pipe.run(\n",
    "    query=\"What year is it?\", params={\"Retriever\": {\"top_k\": 10}, \"Reader\": {\"top_k\": 5}}\n",
    ")\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "pprint(prediction)\n",
    "\n",
    "# Sample output:\n",
    "# {\n",
    "#     'answers': [ <Answer: answer='Eddard', type='extractive', score=0.9919578731060028, offsets_in_document=[{'start': 608, 'end': 615}], offsets_in_context=[{'start': 72, 'end': 79}], document_id='cc75f739897ecbf8c14657b13dda890e', meta={'name': '454_Music_of_Game_of_Thrones.txt'}}, context='...' >,\n",
    "#                  <Answer: answer='Ned', type='extractive', score=0.9767240881919861, offsets_in_document=[{'start': 3687, 'end': 3801}], offsets_in_context=[{'start': 18, 'end': 132}], document_id='9acf17ec9083c4022f69eb4a37187080', meta={'name': '454_Music_of_Game_of_Thrones.txt'}}, context='...' >,\n",
    "#                  ...\n",
    "#                ]\n",
    "#     'documents': [ <Document: content_type='text', score=0.8034909798951382, meta={'name': '332_Sansa_Stark.txt'}, embedding=None, id=d1f36ec7170e4c46cde65787fe125dfe', content='\\n===\\'\\'A Game of Thrones\\'\\'===\\nSansa Stark begins the novel by being betrothed to Crown ...'>,\n",
    "#                    <Document: content_type='text', score=0.8002150354529785, meta={'name': '191_Gendry.txt'}, embedding=None, id='dd4e070a22896afa81748d6510006d2', 'content='\\n===Season 2===\\nGendry travels North with Yoren and other Night's Watch recruits, including Arya ...'>,\n",
    "#                    ...\n",
    "#                  ],\n",
    "#     'no_ans_gap':  11.688868522644043,\n",
    "#     'node_id': 'Reader',\n",
    "#     'params': {'Reader': {'top_k': 5}, 'Retriever': {'top_k': 5}},\n",
    "#     'query': 'Who is the father of Arya Stark?',\n",
    "#     'root_node': 'Query'\n",
    "# }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a079b1f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Query: What year is it?\n",
      "Answers:\n",
      "[   {   'answer': '1963',\n",
      "        'context': '] started running out of names, apparently. I was '\n",
      "                   'born\\\\nSeptember 27th , 1963, in Ohio. So I\\\\u2019m a '\n",
      "                   'Buckeye.\\\\n\\\\nVAN BENSCHOTEN: In what part of Ohio'},\n",
      "    {   'answer': 'e st',\n",
      "        'context': 'f\\\\nsituations that clearly affected what I ultimately '\n",
      "                   'wanted to be, or the studies that I pursued. So I\\\\nthink '\n",
      "                   'from that standpoint, since I was the e'},\n",
      "    {   'answer': 'e\"',\n",
      "        'context': '\",\\n'\n",
      "                   '            \"Medical scientists\",\\n'\n",
      "                   '            \"Science History Institute\"\\n'\n",
      "                   '            \"Jochen Buck was born and grew up in '\n",
      "                   'Reutlingen, Germany, in '},\n",
      "    {   'answer': '1946',\n",
      "        'context': 'esman, January 29, 2009\\\\n\\\\nJG:\\\\n\\\\n2\\\\n\\\\nWhat year was '\n",
      "                   'that?\\\\n\\\\nMG: It was 1946. October 7, '\n",
      "                   '1946.\\\\n\\\\nJG:\\\\n\\\\nWhat was the community of Jersey City '\n",
      "                   'like i'},\n",
      "    {   'answer': 'n the\\\\n\\\\',\n",
      "        'context': '        \"$oid\": \"634d5e9f8577ed72bce8d8b5\"\\n'\n",
      "                   '    \"title\": \"Oral history interview with Jochen Buck\",\\n'\n",
      "                   '            \"Vietnam War (1961-1975)\",\\n'\n",
      "                   '            '}]\n"
     ]
    }
   ],
   "source": [
    "from haystack.utils import print_answers\n",
    "\n",
    "# Change `minimum` to `medium` or `all` to raise the level of detail\n",
    "print_answers(prediction, details=\"minimum\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aba16544",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction2 = pipe.run(\n",
    "    query=\"What did I study?\", params={\"Retriever\": {\"top_k\": 10}, \"Reader\": {\"top_k\": 5}}\n",
    ")\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "pprint(prediction2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "832821c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Query: What did I study?\n",
      "Answers:\n",
      "[   {   'answer': 'respiratory diseases',\n",
      "        'context': 'This was something I was quite interested in \\\\u2013 the '\n",
      "                   'study of respiratory diseases. There \\\\nhad been massive '\n",
      "                   'emigration from the Rhondda with the w'},\n",
      "    {   'answer': 'how to approach Congress people',\n",
      "        'context': 'g aides,\\\\nwho took me in hand and gave me a short course '\n",
      "                   'on how to approach Congress people.\\\\nSo I got, called all '\n",
      "                   'the other project directors, told t'},\n",
      "    {   'answer': 'hereditary hemolytic\\\\nanemias',\n",
      "        'context': 'tivities.\\\\nOver the years I focused my research primarily '\n",
      "                   'on hereditary hemolytic\\\\nanemias, specifically '\n",
      "                   'hemoglobinopathy, but I also had an interest '},\n",
      "    {   'answer': ', etc., a',\n",
      "        'context': 'nwent through the building and they were teaching '\n",
      "                   \"undergraduate English, etc., and I\\\\ncouldn't find any \"\n",
      "                   'cancer work. So I came back to Bethesda and pr'},\n",
      "    {   'answer': 'stry died in Northern',\n",
      "        'context': '  was  of  relatively  little \\\\nimportance, because the '\n",
      "                   'flax industry died in Northern Ireland, the cotton '\n",
      "                   'industry died in Lancashire. So \\\\nit  was  '}]\n"
     ]
    }
   ],
   "source": [
    "from haystack.utils import print_answers\n",
    "\n",
    "# Change `minimum` to `medium` or `all` to raise the level of detail\n",
    "print_answers(prediction2, details=\"minimum\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e7a5a0b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Query: What did I study?\n",
      "Answers:\n",
      "[   {   'answer': 'respiratory diseases',\n",
      "        'context': 'This was something I was quite interested in \\\\u2013 the '\n",
      "                   'study of respiratory diseases. There \\\\nhad been massive '\n",
      "                   'emigration from the Rhondda with the w',\n",
      "        'score': 0.8970402479171753},\n",
      "    {   'answer': 'how to approach Congress people',\n",
      "        'context': 'g aides,\\\\nwho took me in hand and gave me a short course '\n",
      "                   'on how to approach Congress people.\\\\nSo I got, called all '\n",
      "                   'the other project directors, told t',\n",
      "        'score': 0.8005848526954651},\n",
      "    {   'answer': 'hereditary hemolytic\\\\nanemias',\n",
      "        'context': 'tivities.\\\\nOver the years I focused my research primarily '\n",
      "                   'on hereditary hemolytic\\\\nanemias, specifically '\n",
      "                   'hemoglobinopathy, but I also had an interest ',\n",
      "        'score': 0.7782573103904724},\n",
      "    {   'answer': ', etc., a',\n",
      "        'context': 'nwent through the building and they were teaching '\n",
      "                   \"undergraduate English, etc., and I\\\\ncouldn't find any \"\n",
      "                   'cancer work. So I came back to Bethesda and pr',\n",
      "        'score': 0.7526460886001587},\n",
      "    {   'answer': 'stry died in Northern',\n",
      "        'context': '  was  of  relatively  little \\\\nimportance, because the '\n",
      "                   'flax industry died in Northern Ireland, the cotton '\n",
      "                   'industry died in Lancashire. So \\\\nit  was  ',\n",
      "        'score': 0.7388901710510254}]\n"
     ]
    }
   ],
   "source": [
    "from haystack.utils import print_answers\n",
    "\n",
    "# Change `minimum` to `medium` or `all` to raise the level of detail\n",
    "print_answers(prediction2, details=\"medium\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7e7c281",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction2 = pipe.run(\n",
    "    query=\"What did the person work on?\", params={\"Retriever\": {\"top_k\": 12}, \"Reader\": {\"top_k\": 12}}\n",
    ")\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "pprint(prediction2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "96c4f791",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Query: What did the person work on?\n",
      "Answers:\n",
      "[   {   'answer': 'stools',\n",
      "        'context': ' W. Brown]. Everyone\\\\ncalled him \\\\u201cStooly\\\\u201d '\n",
      "                   'Brown. He worked on stools, but he had a lot '\n",
      "                   'of\\\\ncharacter, a lot of interests, and people were ve',\n",
      "        'score': 0.9801039695739746},\n",
      "    {   'answer': 'it to me at all o',\n",
      "        'context': '        \"$oid\": \"633c397c8577ed72bce8d5f5\"\\n'\n",
      "                   '    \"title\": \"Carlson, Virgil 2002 -  Office of NIH '\n",
      "                   'History and Stetten Museum\",\\n'\n",
      "                   '    \"permalink\": \"https://',\n",
      "        'score': 0.9558579921722412},\n",
      "    {   'answer': 'renal failure',\n",
      "        'context': ' The biggest worry I\\\\nhad was my roommate. We were doing '\n",
      "                   'the work on renal failure together,\\\\nCraig Canfield and '\n",
      "                   'I. He had a gun that shot .45 caliber',\n",
      "        'score': 0.921541690826416},\n",
      "    {   'answer': 'rsus a more dis',\n",
      "        'context': '        \"$oid\": \"633c39258577ed72bce8d4b9\"\\n'\n",
      "                   '    \"title\": \"Ingraham, Loring 2002 -  Office of NIH '\n",
      "                   'History and Stetten Museum\",\\n'\n",
      "                   '    \"permalink\": \"https:/',\n",
      "        'score': 0.9084505438804626},\n",
      "    {   'answer': 'bipolar illness',\n",
      "        'context': 'vioral science,\\\\nsometimes more, sometimes less, but in '\n",
      "                   'his work on bipolar illness, for\\\\nexample, he\\\\u2019s '\n",
      "                   'very clear that this is not solely a bioc',\n",
      "        'score': 0.896399974822998},\n",
      "    {   'answer': 'experiments',\n",
      "        'context': 't 8:00\\\\nuntil 1:00, and he wasn\\\\u2019t shuffling papers. '\n",
      "                   'He was doing '\n",
      "                   'experiments.\\\\nWhereas...\\\\n\\\\nPark:\\\\n\\\\n_____ doctoral '\n",
      "                   'students.\\\\n\\\\nRosner:\\\\n\\\\nYeah',\n",
      "        'score': 0.8725752830505371},\n",
      "    {   'answer': 'nFarreras:\\\\n',\n",
      "        'context': 'those were just the two areas that sounded most '\n",
      "                   'interesting to me.\\\\n\\\\nFarreras:\\\\n\\\\nWhat led to that '\n",
      "                   'interest?\\\\n\\\\nStreicher:\\\\n\\\\nThe idea of trying to c',\n",
      "        'score': 0.8452831506729126},\n",
      "    {   'answer': 'ally just\\\\nfelt',\n",
      "        'context': '        \"$oid\": \"633c39958577ed72bce8d64b\"\\n'\n",
      "                   '    \"title\": \"Baldwin, Calvin B. 1997 -  Office of NIH '\n",
      "                   'History and Stetten Museum\",\\n'\n",
      "                   '    \"permalink\": \"https',\n",
      "        'score': 0.821956217288971},\n",
      "    {   'answer': 'research',\n",
      "        'context': 'ttle bit less money, he would take a job at the Clinical '\n",
      "                   \"Center and do research\\\\nat the same time. I'm not sure \"\n",
      "                   'how popular that is today, but at that',\n",
      "        'score': 0.8202618956565857},\n",
      "    {   'answer': 'could b',\n",
      "        'context': '        \"$oid\": \"633c394c8577ed72bce8d541\"\\n'\n",
      "                   '    \"title\": \"Gallo, Robert 1994 A -  Office of NIH '\n",
      "                   'History and Stetten Museum\",\\n'\n",
      "                   '    \"permalink\": \"https://',\n",
      "        'score': 0.811424732208252},\n",
      "    {   'answer': 'Ungerleider: I think so,',\n",
      "        'context': '        \"$oid\": \"633c38a68577ed72bce8d2d9\"\\n'\n",
      "                   '    \"title\": \"Ungerleider, Leslie 2002 -  Office of NIH '\n",
      "                   'History and Stetten Museum\",\\n'\n",
      "                   '    \"permalink\": \"http',\n",
      "        'score': 0.7877588272094727},\n",
      "    {   'answer': 'behavioral aspects of aging',\n",
      "        'context': '\\\\nFarreras:\\\\n\\\\nOh, so it was just those people working '\n",
      "                   'on the behavioral aspects of aging that\\\\nwent to this '\n",
      "                   'Aging Section in the Lab of Psychology at',\n",
      "        'score': 0.7687078714370728}]\n"
     ]
    }
   ],
   "source": [
    "from haystack.utils import print_answers\n",
    "\n",
    "# Change `minimum` to `medium` or `all` to raise the level of detail\n",
    "print_answers(prediction2, details=\"medium\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c366e46",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}