paperExtract_final_sdk_addn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob, re, sys, os, ssl, unicodedata, itertools, lxml, bs4, requests, multiprocessing\n",
    "import pandas as pd\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from dataclasses import dataclass\n",
    "from pathlib import Path\n",
    "from requests.adapters import HTTPAdapter\n",
    "from requests.packages.urllib3.util.retry import Retry\n",
    "from multiprocessing.pool import Pool\n",
    "#from grobid_client_python import grobid_client as grobid\n",
    "from bs4 import BeautifulSoup\n",
    "import boto3\n",
    "\n",
    "\n",
    "##########\n",
    "# This script allows for conversion of pdfs to an excel spreadsheet, parsing out text by keywords from each file\n",
    "# Written/modified by Corinn Small, corinn.small@ucsf.edu\n",
    "#########\n",
    "\n",
    "\n",
    "def convert_text(inpath,outpath):\n",
    "    \n",
    "    '''\n",
    "    converts text from pdf to tei.xml using grobid web service api, remember computer has to be connected to the server: cd grobid-0.6.1/ -> ./gradlew run\n",
    "    input: path to papers\n",
    "    output: tei.xml file per pdf\n",
    "    '''\n",
    "\n",
    "    print('Converting text...')\n",
    "    print('')\n",
    "    \n",
    "    # make sure your aws role has the correct permissions to run textract, see https://docs.aws.amazon.com/textract/latest/dg/api-async-roles.html to configue \n",
    "    \n",
    "    client = boto3.client('textract')\n",
    "    for m in dir(client): print(m)  # what are the function attached to the client object?... _ = 'private' function meaning dont call it\n",
    "    \n",
    "    \n",
    "    #client.StartDocumentTextDetection()  #for detecting and analyzing text in multipage docs (asynchronous op), I would try both syncrhonous and asynchronous, maybe the synchronous works for pdfs with just a couple of pages?\n",
    "\n",
    "#have to set up a aws account s3 bucket... (cloud object storage)\n",
    "#for s3object, you need to specify bucket, name and version:\n",
    "    #s3_obj =  {\"Bucket\": BUCKET, \"Name\": NAME}  #creating dictionary obj\n",
    "\n",
    "    #response = client.GetDocumentTextDetection(Document = {'S3Object': s3_obj})  #nesting dictionary s3_obj inside document dictionary and giving that to the client function \n",
    "#get results returned, we want an s3object dictionary to be returned \n",
    "\n",
    "#response is a dictionary type: you can look at the document page that specifies the basic the s3 object metadata to get fields that you can query from the dict\n",
    "\n",
    "    #detect = response.copy()  #save output as detection (of the pdf output)\n",
    "\n",
    "#now to analyze all the gibberish (output = list of block objects) this is the synchronous version:\n",
    "    #response2 = client.analyze_document(Document = {'S3Object': s3_obj}, FeatureTypes = ['TABLES'])\n",
    "#document = s3object\n",
    "#observation: of required syntax: the layout is confusing to you, the request syntax that the boto3 doc specifies is telling you what type of ojbect to pass the client.function and the naming convention of the returned object! thats it! dont worry, just keep in mind the indendation and parentheses\n",
    "#featuretypes is required\n",
    "    #response2['Blocks'][0].keys()  # print name of each block\n",
    "\n",
    "    #for b in response2['Blocks']:\n",
    "        #if b['BlockType'] == 'LINE':\n",
    "            #print(\"{}\\t{}\".format(b['Text'], b['Confidence']))  # for everytime line is found, what is that word and the confidence at which its identified, it's important/critical to check the confidence scores!\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "    #client = grobid.grobid_client(config_path=\"./grobid_client_python/config.json\")\n",
    "    #client.process(\"processFulltextDocument\", inpath, outpath)\n",
    "    \n",
    "    print('Done')\n",
    "\n",
    "def read_tei(tei_file):\n",
    "    '''\n",
    "    Reads in an xml file and returns a beautifulsoup object\n",
    "    \n",
    "    '''\n",
    "    with open(tei_file, 'r') as tei:\n",
    "        soup = BeautifulSoup(tei, 'html.parser')\n",
    "        return soup\n",
    "    raise RuntimeError('Cannot generate a soup from the input')\n",
    "    \n",
    "\n",
    "def elem_to_text(elem, default='NA'):\n",
    "    '''\n",
    "    Returns element if it exists, if not returns NA\n",
    "    \n",
    "    '''\n",
    "    if elem:\n",
    "        return elem.getText()\n",
    "    else:\n",
    "        return default\n",
    "    \n",
    "\n",
    "#create class for storing pdf info\n",
    "\n",
    "class TEIFile(object):\n",
    "    def __init__(self, filename):\n",
    "        self.filename = filename\n",
    "        self.soup = read_tei(filename)  #creates soup object\n",
    "        self._text = None\n",
    "        self._title = ''  \n",
    "        self._abstract = ''\n",
    "        self._keytext = {}\n",
    "        self._check = None\n",
    "        \n",
    "    @property\n",
    "    def doi(self, id_='DOI'):\n",
    "        '''\n",
    "        retrieve id\n",
    "        '''\n",
    "    \n",
    "        idno_elem = self.soup.find('idno', type='DOI')\n",
    "        if not idno_elem:\n",
    "            return 'no id'\n",
    "        else:\n",
    "            return idno_elem.getText()\n",
    "    \n",
    "    @property\n",
    "    def title(self):\n",
    "        '''\n",
    "        retrieve title\n",
    "        '''\n",
    "        if not self._title:\n",
    "            self._title = self.soup.title.getText()\n",
    "        return self._title\n",
    "    \n",
    "    @property\n",
    "    def abstract(self):\n",
    "        '''\n",
    "        retrieve abstract\n",
    "        '''\n",
    "        \n",
    "        if not self._abstract:\n",
    "            abstract = self.soup.abstract.getText(separator=' ', strip=True)\n",
    "            self._abstract = abstract\n",
    "        return self._abstract\n",
    "      \n",
    "    @property\n",
    "    def authors(self):\n",
    "        '''\n",
    "        retrieve authors\n",
    "        '''\n",
    "        authors_in_header = self.soup.analytic.find_all('author')\n",
    "\n",
    "        result = []\n",
    "        \n",
    "        @dataclass\n",
    "        class Person:\n",
    "            firstname: str\n",
    "            middlename: str\n",
    "            surname: str\n",
    "                \n",
    "        for author in authors_in_header:\n",
    "            persname = author.persname\n",
    "            if not persname:\n",
    "                continue\n",
    "            firstname = elem_to_text(persname.find(\"forename\", type=\"first\"))\n",
    "            middlename = elem_to_text(persname.find(\"forename\", type=\"middle\"))\n",
    "            surname = elem_to_text(persname.surname)\n",
    "            person = Person(surname, firstname, middlename)\n",
    "            result.append(person)\n",
    "        return result\n",
    "    \n",
    "    @property\n",
    "    def text(self):\n",
    "        '''\n",
    "        retrieves text\n",
    "        returns dictionary by subsection\n",
    "        '''\n",
    "        #print(self.soup.prettify())\n",
    "        print(self.filename.split('/')[-1])\n",
    "        print('')\n",
    "        \n",
    "        if not self._text:\n",
    "            paper_text = {}  \n",
    "                \n",
    "            divs = self.soup.body.find_all('div')\n",
    "            #print(divs)\n",
    "                \n",
    "            for div in divs:   \n",
    "                if not div.get('type'):  # if div is neither an appendix nor reference\n",
    "                    heads = div.find_all('head')  #find all subsections\n",
    "                    #print(heads)\n",
    "                    \n",
    "                    if not heads:  #if there aren't any subsections\n",
    "                        sect = []\n",
    "                        for p in div.find_all('p'):  #add all sentences to one section titled 'body'\n",
    "                            #print(p)\n",
    "                            sect_text = p.get_text(separator=' ', strip=True)\n",
    "                            sect.append(sect_text)\n",
    "                            paper_text['body'] = sect  \n",
    "                        \n",
    "                        #print(sect)\n",
    "                        \n",
    "                    else:  #otherwise for each subsection create a new list with corresponding text and add it to a div dictionary\n",
    "                        for head in heads:\n",
    "                            sect_ = []\n",
    "                            head_text = head.get_text(separator=' ', strip=True).lower()\n",
    "                            \n",
    "                            \n",
    "                            if len(head_text.split(' ')) > 7:  #if the head title is < 7 words long (arbitrary #) ask the user to clarify its validity\n",
    "                                print('subtitle sentence: ', head_text)\n",
    "                                \n",
    "                                ans = input('Is this subsection a full sentence? y/n')\n",
    "                                ok_ans = ('y','n')\n",
    "                                \n",
    "                                while ans not in ok_ans:\n",
    "                                    print('Y or N only please!')\n",
    "                                    ans = input('Is this subsection a full sentence? y/n')\n",
    "\n",
    "                                if ans == 'y':  #if the title is a sentence\n",
    "                                    print('Including subsection title in searchable text, but still include it as a separate subsection...')\n",
    "                                    print('')\n",
    "                                    sect_.append(head_text)  #include it in the text\n",
    "                                    for p in div.find_all('p'):  #for each paragraph find all text \n",
    "                                        #print(p)\n",
    "                                        sect_text = p.get_text(separator=' ', strip=True)\n",
    "                                        sect_.append(sect_text)  #add it to the list\n",
    "\n",
    "\n",
    "                                elif ans == 'n':  #if the title is not a sentence\n",
    "                                    ans1 = input('Does this subsection make sense? y/n')  #does the title make sense in general?\n",
    "\n",
    "                                    while ans1 not in ok_ans:\n",
    "                                        print('Y or N only please!')\n",
    "                                        ans1 = input('Is this subsection a full sentence? y/n')\n",
    "\n",
    "\n",
    "                                    if ans1 == 'y':  #if the title makes sense treat it as a regular subsection\n",
    "                                        print('Valid subsection...')\n",
    "                                        print('')\n",
    "\n",
    "                                        for p in div.find_all('p'):\n",
    "                                            #print(p)\n",
    "                                            sect_text = p.get_text(separator=' ', strip=True)\n",
    "                                            sect_.append(sect_text)\n",
    "\n",
    "                                    elif ans1 == 'n':  #if the title doesn't make sense, mark self._check True so the user knows to check it later\n",
    "                                        print('Flagging nonsense...')\n",
    "                                        print()\n",
    "\n",
    "                                        self._check = True\n",
    "\n",
    "                                        for p in div.find_all('p'):  #still include the subsection's text in list for keyword searching\n",
    "                                            #print(p)\n",
    "                                            sect_text = p.get_text(separator=' ', strip=True)\n",
    "                                            sect_.append(sect_text)\n",
    "\n",
    "                                    \n",
    "\n",
    "                            else:  #if the subsection title is not longer than 7 words, find all text and add it to the list\n",
    "                                for p in div.find_all('p'):\n",
    "                                    #print(p)\n",
    "                                    sect_text = p.get_text(separator=' ', strip=True)\n",
    "                                    sect_.append(sect_text)\n",
    "                                    #print('subsection title < 7 words')\n",
    "                        \n",
    "                                \n",
    "                            \n",
    "                            paper_text[head_text] = sect_ \n",
    "                                    \n",
    "                                    \n",
    "                        #for figure descriptions....(WORK IN PROGRESS)\n",
    "                            if head_text.find('fig') != -1:  #if figure is treated as its own subsection, find all text and include it in its own list\n",
    "                                print('Figure treated as subsection: ', head_text)\n",
    "                    \n",
    "                                for p in div.find_all('p'):\n",
    "                                    #print(p)\n",
    "                                    sect_text = p.get_text(separator=' ', strip=True)\n",
    "                                    sect_.append(sect_text)\n",
    "                                    paper_text['figure'] = sect_\n",
    "                                \n",
    "                            for fig in self.soup.body.find_all('figure'):   #for figure sections find the figure descriptions and print it\n",
    "                                figDesc = fig.find_all('figDesc')\n",
    "                                for f in figDesc:\n",
    "                                    print('Figure Description: ',f)\n",
    "                                           \n",
    "                  \n",
    "                \n",
    "            #if paper is short, all the text might show up in the abstract section of the xml file instead of div\n",
    "            if not divs:\n",
    "                self._check = True\n",
    "                \n",
    "                sect = []\n",
    "                ps = self.soup.abstract.find_all('p')\n",
    "                \n",
    "                if not ps:\n",
    "                    sect_text = self.soup.abstract.get_text(separator=' ', strip=True)\n",
    "                    sect.append(sect_text)\n",
    "\n",
    "\n",
    "                else:\n",
    "                    for p in ps:\n",
    "                        sect_text = p.get_text(separator=' ', strip=True)\n",
    "                        sect.append(sect_text)\n",
    "\n",
    "                paper_text['body'] = sect   #add all sentences to one section titled 'body'\n",
    "                \n",
    "                    \n",
    "                \n",
    "                \n",
    "                \n",
    "            self._text = paper_text\n",
    "        return self._text\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    def keytext(self, keywords):\n",
    "        '''\n",
    "        retrieves sentences by keyword\n",
    "        returns dictionary\n",
    "        '''\n",
    "        \n",
    "        \n",
    "        for keyword in keywords:\n",
    "            self._keytext[keyword] = []\n",
    "            \n",
    "        \n",
    "        for k,v in self._text.items():  #for subsection and text \n",
    "                for keyword in keywords:  #search thru sentences \n",
    "                    sub_dict = {}\n",
    "                    section = k.lower()\n",
    "                    sub_dict[section] = []  #create list for each subsection in dictionary\n",
    "                    \n",
    "                    for i in v:  #for paragraph in text\n",
    "                        sentences = sent_tokenize(i)  #get list of sentences\n",
    "                        sentences = [s.lower() for s in sentences]\n",
    "                        \n",
    "                    \n",
    "                        for sentence in sentences:\n",
    "                            result = re.findall('\\\\b' + keyword + '\\\\b', sentence)  #find keyword in sentence\n",
    "\n",
    "                            if len(result) > 0:  #if keyword exists, \n",
    "                                sub_dict[section].append(sentence)  #add sentence to subsection list\n",
    "\n",
    "                            else:  \n",
    "                                pass\n",
    "                    \n",
    "                    #print(sub_dict)\n",
    "                    self._keytext[keyword].append(sub_dict)  #adds each subsection to keyword dictionary\n",
    "                        \n",
    "        return self._keytext                   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get input from user\n",
    "\n",
    "cases = 'summary'\n",
    "path = '/Users/corinnsmall/Documents/BPDCN/bpdcn_papers/' + cases +'_case_papers/'\n",
    "key = 'wgs'\n",
    "output = 'output_' + cases + '_cases_' + key\n",
    "\n",
    "associated_diseases_keywords = ['acute myeloid leukemia', 'AML', 'acute lymphoblastic leukemia', 'ALL', 'leukemia', 'non-hodgkin lymphoma', 'Hodgkin lymphoma',\n",
    "                                'lymphoma','myelogenous leukemia', 'multiple myeloma', 'myeloma', 'chronic myelogenous leukemia', 'CML', 'chronic lymphocytic leukemia', \n",
    "                                'CLL', 'carcinoma','Pleuropulmonary blastoma', 'blastoma', 'neuroblastoma', 'melanoma', 'sarcoma', 'skin cancer', 'hairy cell leukemia',\n",
    "                                'ependymoma', 'chordoma','bone cancer', 'bladder', 'AIDS-related lymphoma', 'thyroid cancer', 'colon cancer', 'rectal cancer',\n",
    "                                'prostate cancer', 'chronic myeloid leukemia', 'myeloproliferative', 'myelodysplastic', 'mast cell', 'mastocytosis', 'lymphoblastic', \n",
    "                                'follicular lymphoma', 'marginal zone lymphoma', 'langerhan', 'polycythemia vera', 'essential thrombocythemia', 'myelofibrosis', \n",
    "                                'mycosis fungoides', 'sezary', 'burkitt', 'cmml', 'chronic myelomonocytic leukemia']\n",
    "\n",
    "morphology_keywords = ['Vacuoles','Vacuolated','Microvacuoles','Lymphoid','Eccentrically','Eccentric','Prominent nucleoli', 'Small nucleoli','Large nucleoli',\n",
    "                       'Medium nucleoli','azurophilic','Blast','Blastoid','Agranular','Basophilic','Eosinophilic','Perivascular','Periadnexal','Pseudopodia','Hairy','Rosary beads',\n",
    "                       'Large nucleolus','Monoblastic','Monocytic','Histiocytic','Histiocytoid','Small granules','Large granules','Granulated','Granular','Condensed chromatin',\n",
    "                       'Dispersed chromatin','Fine chromatin','Pale cytoplasm','Poorly differentiated', 'Large sized','Medium sized','Small sized','Plasmablast', \n",
    "                       'Plasmacytoid','plasmacytic','Immature','lymphoblast']\n",
    "\n",
    "nuclei_keywords = ['convoluted nuclei','convoluted','convolutions','nuclear folds','folds','membrane irregularities','slightly irregular','round nuclei','oval nuclei','vesicular chromatin','cleaved nuclei']\n",
    "\n",
    "\n",
    "gene_keywords = ['ABL1','AKT1','ALK','APC','ARF','ARID1A','ASH1L','ASXL1','ASXL3','ATM','ATR','AXIN2','BRAF','BCORL1','CAL-1','CCND3','CEBPA','CHD8','CHP2','CDH1','CDKN2A',\n",
    "                 'CDKN1B','CDKN2B','CIC','CREBBP','CSF1R','CTNNB1','CPS1','CROCC','CXCR4','DAXX','DIP2A','DNMT3A','EGFR','EGR1','EP300','ERBB2','ERBB4','ERCC4','ETV6','EYA2',\n",
    "                 'EZH2','FBXW7','FGFR1','FGFR2','FGFR3','FLT3','FLT3-ITD','FLT3-other','GNA11','GNAS','GNAQ','GPR160','HES6','HNF1A','HOXB9','HRAS','IDH1','IDH2','IKZF1',\n",
    "                 'IKZF2','IKZF3','IVL','JAK2','JAK3','KDM40','KDR','KIT','cKIT','KRAS','MAD1L1','MAPK1','MCL1','MET','MLH1','MLL','MLL2','MLL3','MPL','MSH6','MYB','MYBL1',\n",
    "                 'MYC','MYST3','MYST4','NF1','NOTCH1','NPM1','NR3C1','NRAS','PALB2','PARK2','PBRM1','PDGFRA','PHF2','PHF6','PIK3CA','PMDC05','PLCXD3','PLP1','PTEN','PTPN11',\n",
    "                 'PTPN23','PVT1','RAD52','RANBP2','RAS','RB1','RET','RFPL1','RHOA','RUNX1','RUNX2','SARDH','SIGLEC6','SLC25A10','SMAD4','SMARCB1','SMARCD1','SMO','SRC','SRCAP',\n",
    "                 'SRSF2','STK3','STK11','SUPT3H','SUZ12','TCF3','TCL1A','TEL','TET2','TRMT61B','TP53','U2AF1','UBE2G2','VHL','WNT3','WNT7B','WNT10A','WT1','ZEB2','ZRSR2']\n",
    "\n",
    "wgs_keywords = ['whole exome sequencing', 'whole genome sequencing', 'wes', 'wgs', 'sequencing', 'exome', 'whole genome', 'whole-genome', 'whole-exome', 'genome-wide']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "keylist = [i.lower() for i in wgs_keywords]  #make all keywords lower case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Converting text...\n\n_PY_TO_OP_NAME\n__class__\n__delattr__\n__dict__\n__dir__\n__doc__\n__eq__\n__format__\n__ge__\n__getattr__\n__getattribute__\n__gt__\n__hash__\n__init__\n__init_subclass__\n__le__\n__lt__\n__module__\n__ne__\n__new__\n__reduce__\n__reduce_ex__\n__repr__\n__setattr__\n__sizeof__\n__str__\n__subclasshook__\n__weakref__\n_cache\n_client_config\n_convert_to_request_dict\n_emit_api_params\n_endpoint\n_exceptions\n_exceptions_factory\n_get_waiter_config\n_load_exceptions\n_loader\n_make_api_call\n_make_request\n_register_handlers\n_request_signer\n_response_parser\n_serializer\n_service_model\nanalyze_document\ncan_paginate\ndetect_document_text\nexceptions\ngenerate_presigned_url\nget_document_analysis\nget_document_text_detection\nget_paginator\nget_waiter\nmeta\nstart_document_analysis\nstart_document_text_detection\nwaiter_names\nDone\n"
     ]
    }
   ],
   "source": [
    "#convert pdfs to xml, need to connect to grobid api to use convert_text()\n",
    "\n",
    "path = '/Users/corinnsmall/Documents/BPDCN/bpdcn_papers/'\n",
    "outpath = path + 'xml_output/'\n",
    "convert_text(path,outpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "laribi_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "leclerc_2017.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "hamadeh_2019.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "martinmartin_2015.tei.xml\n",
      "\n",
      "subtitle sentence:  immunophenotypic subgroups of blastic plasmacytoid dendritic cell neoplasms\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  clinical and laboratory features of the distinct maturation-associated subgroups of bpdcn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "jaye_2006.tei.xml\n",
      "\n",
      "subtitle sentence:  bdca-2 is expressed on a subset of cd4 þ cd56 þ hematodermic neoplasms\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "moreno_2013.tei.xml\n",
      "\n",
      "subtitle sentence:  immunohistochemical expression in ffpe samples and image capture\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  spib protein expression in major mature b-and t-cell lymphoma subtypes\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  spib protein expression in bpdc neoplasms and their potential mimics\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "rizvi_2012.tei.xml\n",
      "\n",
      "subtitle sentence:  expression of the cd2ap adaptor molecule in normal, reactive and neoplastic human tissue\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "lee_2019.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "wang_2020.tei.xml\n",
      "\n",
      "subtitle sentence:  differential immunophenotypic characteristics of bpdcn and reactive pdcs\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  establishment and validation of a flow cytometry assay for minimal residual disease\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  flow cytometry versus immunohistochemistry in the assessment of minimal residual disease\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "tsunoda_2012.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "sapienza_2014.tei.xml\n",
      "\n",
      "subtitle sentence:  bpdcns mirror myeloid resting pdcs and are distinct from other acute leukemias\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  bpdcns differ from non-neoplastic pdcs for several specific genes and cellular programs\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  nf-kb pathway is a candidate therapeutic target in bpdcn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "facchetti_2008.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "martin_2006.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "martin_2016.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "nomburg_2020.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "benet_2011.tei.xml\n",
      "\n",
      "subtitle sentence:  de novo mlcs show a specific histologic phenotype\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  does the immunohistochemical panel allow diagnosis of mlc and distinction of types of myeloid disorders?\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  chronic mpss cannot be distinguished from other underlying myeloid disorders\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "feuillard_2002.tei.xml\n",
      "\n",
      "subtitle sentence:  immunophenotypic profile of cd4 ؉ cd56 ؉ malignant cells\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  immunophenotypic characterization of cd4 ؉ cd56 ؉ tumor cells\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "sun_2018.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "petrella_2010.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "sukswai_2019.tei.xml\n",
      "\n",
      "subtitle sentence:  staining characteristics of tcf4/cd123 dual-color immunostain in bpdcn cases\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  comparison of tcf4/cd123 dual-color immunohistochemistry staining to multiparameter flow cytometry in bone marrow samples\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  staining characteristics of tcf4/cd123 dual-color immunohistochemistry in non-bpdcn tissues\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "ferreira_2016.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "tang_2018.tei.xml\n",
      "\n",
      "subtitle sentence:  conventional chromosomal analysis and fish analysis on bone marrow samples\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "saczonek_2018.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "yu_2015.tei.xml\n",
      "\n",
      "subtitle sentence:  cd56 + dcs clustered together with mdcs but not pdcs by transcriptomic analysis\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  cd56 + dcs are functionally analogous to mdcs\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  bpdcn is closely related to cd56 + dcs, but not pdcs by global gene expression profiling\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  facs analysis of dc subsets in human blood\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "child_2003.tei.xml\n",
      "\n",
      "subtitle sentence:  analysis of t-cell receptor c gene and immunoglobulin heavy chain gene rearrangements\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "ohe_2018.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "tsagarakis_2010.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "hashikawa_2011.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "murashige_2005.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "bruggen_2020.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "roosweil_2013.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "deng_2017.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "petrella_2002.tei.xml\n",
      "\n",
      "subtitle sentence:  npm is a rare population of cd56+ cells related to plasmacytoid monocytes\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "mori_2001.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "pemmaraju_2019_nejm.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "johnson_2016.tei.xml\n",
      "\n",
      "subtitle sentence:  comparison of mnda expression with other markers of myelomonocytic differentiation in emls\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  mnda expression in cd4+/cd56+ and cd4+/ cd56à emls\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "dabaja_2017.tei.xml\n",
      "\n",
      "subtitle sentence:  study objectives, definition of endpoints, and statistical methodology\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  engraftment kinetics, gvhd (acute and chronic), nrm and relapse/progression\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "emadali_2016.tei.xml\n",
      "\n",
      "subtitle sentence:  glucocorticoid response element and e2f reporter assays and rna interference\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  results 5q anomalies in bpdcn confer adverse clinical outcome\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  targeting of nr3c1 by 5q anomalies in bpdcn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  loss-of-ezh2 function is a hallmark of 5q alterations that target nr3c1 in bpdcn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "massone_2004.tei.xml\n",
      "\n",
      "subtitle sentence:  cutaneous medium/large pleomorphic t-cell lymphoma, not otherwise specified\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "marmouset_2019.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "pagano_2013.tei.xml\n",
      "\n",
      "subtitle sentence:  © f e r r a t a s t o r t i f o u n d a t i o n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "gilis_2012.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "park_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "tzankov_2017.tei.xml\n",
      "\n",
      "subtitle sentence:  plasmacytoid dendritic cell proliferations and mature plasmacytoid dendritic cell neoplasm in the bone marrow\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  blastic plasmacytoid dendritic cell neoplasms in the bone marrow\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "fujii_2020.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "bekkenk_2004.tei.xml\n",
      "\n",
      "subtitle sentence:  comparison between blastic nk cutaneous cd561 myeloid leukemia\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "marafioti_2008.tei.xml\n",
      "\n",
      "subtitle sentence:  detection of novel markers associated with normal pdcs\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "boddu_2018.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "dorfman_2010.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "karube_2003.tei.xml\n",
      "\n",
      "subtitle sentence:  no. classification cd2 cd3 cd4 cd7 cd8 cd13 cd20 cd33 cd123 cd34 cd56 cd68 mpo tdt eb-ish\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "sangle_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "griffin_2017.tei.xml\n",
      "\n",
      "subtitle sentence:  and b.3). leucostasis complicating a compromised cerebrovascular circulation may have been the patho-biology behind clinical presentation.\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "lozzi_2006.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "julia_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "gruson_2013.tei.xml\n",
      "\n",
      "subtitle sentence:  l-asparaginase with methotrexate and dexamethasone is an effective treatment combination in blastic plasmacytoid dendritic cell neoplasm\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "jen_2019.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "dietrich_2011.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "heinicke_2015.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "jardin_2011.tei.xml\n",
      "\n",
      "subtitle sentence:  tet2 and tp53 mutations are frequently observed in blastic plasmacytoid dendritic cell neoplasm\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "fiandrino_2013.tei.xml\n",
      "\n",
      "subtitle sentence:  g . fi a n d r i n o m. a r r a 1 r. r i b o\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "urosevic_2005.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "khwaja_2016.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "deotare_2015.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "luicioni_2012_blood.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "nguyen_2015.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "xu_2016.tei.xml\n",
      "\n",
      "subtitle sentence:  evaluation and scoring of ihc staining of pd-l1 and pd-l2\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  expression of pd-l1 and pd-l2 in histiocytes and dendritic cell subsets\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  pd-1 ligand expression in non-neoplastic histiocytic and dendritic cell disorders\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  pd-1 ligand expression in malignant histiocytic and dendritic cell neoplasms\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  comparison of pd-l1 and pd-l2 expression between hs, ids, and fds\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "betrian_2017.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "ottou_2019.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "ottou_2009.tei.xml\n",
      "\n",
      "subtitle sentence:  bdca-2 expression is strongly associated with a pdcl phenotype\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  cd135 and dc maturation marker expression does not distinguish pdcl from other leukaemia\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "shapiro_2003.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "hamada_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "vitte_2012.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "suzuki_2017_leu.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "sweet_2020.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "jacob_2003.tei.xml\n",
      "\n",
      "subtitle sentence:  characterization of the malignant cell cd4 + 56 + lin − malignant cells arise from pdc\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  © f e r r a t a s t o r t i f o u n d a t i o n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  tumor cell affiliation to the pdc lineage relies on immunophenotypic criteria\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  © f e r r a t a s t o r t i f o u n d a t i o n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  cd4 + cd56 + lin − pdc are arrested at an early stage of maturation\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  © f e r r a t a s t o r t i f o u n d a t i o n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  which origin for malignant cd4 + cd56 + lin − dc: lymphoid or myeloid?\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "Figure treated as subsection:  histologic findings (figure 4)\n",
      "subtitle sentence:  © f e r r a t a s t o r t i f o u n d a t i o n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "lucioni_2011.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "kawamata_2005.tei.xml\n",
      "\n",
      "subtitle sentence:  m u m u m u m u m u d a u d i n o r m a l p a t i e n t 3 p a t i e n t 4 p a t i e n t 5\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "dijkman_2007.tei.xml\n",
      "\n",
      "subtitle sentence:  novel membranebound markers for cd4 ؉ cd56 ؉ hn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  detailed analysis of chromosomal regions with recurrent deletion in cd4 ؉ cd56 ؉ hn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  confirmation of microarray expression data by real-time pcr\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "magro_2009.tei.xml\n",
      "\n",
      "subtitle sentence:  cd4 1 cd56 1 malignancy associated with t-cell clonality (nk t-cell lymphoma, anaplastic large cell lymphoma, and mycosis fungoides) nk t-cell lymphoma (cases 1 and 2)\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "bulbul_2018.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "kleppe_2016.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "ulrickson_2017.tei.xml\n",
      "\n",
      "subtitle sentence:  autoimmune hemolytic anemia in a young man with acute hepatitis e infection\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "safaei_2019.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "menezes_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "martinez_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "jegalian_2010.tei.xml\n",
      "\n",
      "subtitle sentence:  clinical features of the national cancer institute cases and previously published cases\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  patient n. cd4 cd56 tdt cd123 cd303* cd7 cd3 cd68 cd117 mpo other\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "gera_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "ascani_2008.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "rakozy_2001.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "pilichowska_2007.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "delettre_2012.tei.xml\n",
      "\n",
      "subtitle sentence:  immunostaining and flow cytometry analysis on normal peripheral blood and acute leukemia cell samples\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  expression of tcl1 and ilt7 in normal blood cells\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  tcl1 expression on plasmacytoid denditic cell leukemia and other acute leukemia\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "stenzinger_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "testa_2014.tei.xml\n",
      "\n",
      "subtitle sentence:  cd123 expression on human hematopoietic stem/ progenitor cells\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "agha_2019.tei.xml\n",
      "\n",
      "subtitle sentence:  case records of the massachusetts general hospital (case\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  hospital of the university of pennsylvania philadelphia, pa\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "kim_2015.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "kim_2005.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "rauh_2012.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "pemmaraju_2019.tei.xml\n",
      "\n",
      "subtitle sentence:  case records of the massachusetts general hospital (case\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  hospital of the university of pennsylvania philadelphia, pa\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "ji_2009.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "bayerl_2002.tei.xml\n",
      "\n",
      "subtitle sentence:  a b s t r a c t only a few blastic natural killer (nk) cell leukemias and lymphomas have been reported. as such, the clinicopathologic spectrum of this disease is incompletely understood.\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "demiroz_2020.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "sapienza_2019.tei.xml\n",
      "\n",
      "subtitle sentence:  more bioinformatics details are provided in the online supplementary appendix and online\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  whole-exome sequencing reveals the epigenetic program dysregulation as the main theme of the blastic plasmacytoid dendritic cell neoplasm mutational landscape\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  blastic plasmacytoid dendritic cell neoplasm transcriptome profiling confirms the dysregulation of epigenetic programs\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  genome-wide chip-sequencing substantiates epigenetic dysregulation of cell cycle program in blastic plasmacytoid dendritic cell neoplasms\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  in vivo blastic plasmacytoid dendritic cell neoplasm modeling demonstrates combined epigenetic therapy as effective in controlling disease progression\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "lezama_2019.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "eros_2009.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "subramanian_2018.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "cronin2012.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "herling_2003.tei.xml\n",
      "\n",
      "subtitle sentence:  cd123 and tcl1 expression is characteristic of cd4 ؉ cd56 ؉ bts\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  table 1. tcl1 expression as detected by immunostaining and western blot\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  tcl1 is highly expressed in cd123 ؉ lymph node plasmacytoid dendritic cells (dc2s)\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  tcl1 is negative in true nk-cell lymphomas, mature t-cell malignancies, and myeloid leukemias\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  tcl1 expression does not correlate with levels of pakt\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "oshimi_2013.tei.xml\n",
      "\n",
      "subtitle sentence:  clinical and laboratory features of patients with definite diagnosis\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  comparison of precursor nk-cell all and blastic nk-cell lymphoma\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  is aggressive nk-cell leukemia/lymphoma a leukemic phase of extranasal nk-cell lymphoma?\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "salva_2014.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "yamada_2000.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "hwang_2013.tei.xml\n",
      "\n",
      "subtitle sentence:  p a t i e n t s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  i m m u n o h i s t o c h e m i s t r y ( i h c )\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "borchiellini_2013.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "alfayez_2019.tei.xml\n",
      "\n",
      "subtitle sentence:  investigation into possible mechanisms of resistance in bpdcn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "cernan_2020.tei.xml\n",
      "\n",
      "subtitle sentence:  pt. common bpdcn markers myeloid, b and t-lymphoid lineage specific markers\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "aung_2019.tei.xml\n",
      "\n",
      "subtitle sentence:  evaluation of pd1 and pd-l1 expression by immunohistochemistry\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "torres_2019.tei.xml\n",
      "\n",
      "subtitle sentence:  | detection of genomic rearrangements and fusion transcripts\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  the output was subjected to a wilcoxon rank test and a kolmogorov-\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  | validation of structural genomic alterations and small-scale mutations\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  | structural alterations involving ikzf1 are recurrent in bpdcn\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "suzuki_2017.tei.xml\n",
      "\n",
      "subtitle sentence:  p a t i e n t s a m p l e s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  h i s t o p a t h o g y a n d i m m u n o p h e n o t y p i c a l a n a l y s i s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  r e s p o n s e t o t r e a t m e n t\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  s t a t i s t i c a l a n a l y s i s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  p a t i e n t c h a r a c t e r i s t i c s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  h i s t o p a t h o l o g i c a l , i m m u n o p h e n o t y p i c a l a n d k a r y o t y p i c a l f e a t u r e s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  d u a l p r i m a r y m a l i g n a n c i e s a n d o t h e r d i s e a s e\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  c o m p a r i s o n o f t d t -p o s i t i v e a n d t d t -n e g a t i v e c u t a n e o u s b p d c n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  c o m p a r i s o n o f n o d u l a r a n d d i s s e m i n a t e d t y p e s o f c u t a n e o u s b p d c n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "agha_2018.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "wiesner_2010.tei.xml\n",
      "\n",
      "subtitle sentence:  loss of chromosome region 12p13 and the weak expression of p27 kip1\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  loss of chromosome 9 and no expression of p16 ink4a\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "kazakov_2003.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "julia_2013.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "arora_2013.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "paluri_2015.tei.xml\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "sakamoto_2018.tei.xml\n",
      "\n",
      "subtitle sentence:  recurrent 8q24 rearrangement and myc expression, and the association with cytomorphology\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  gene expression patterns of myc + bpdcn and myc\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n",
      "lezama_2018.tei.xml\n",
      "\n",
      "subtitle sentence:  c a s e s e l e c t i o n\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  c y t o g e n e t i c a n a l y s i s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  l i t e r a t u r e r e v i e w\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  c a s e r e p o r t s\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "subtitle sentence:  r e v i e w o f c a s e s p u b l i s h e d i n t h e l i t e r a t u r e\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Is this subsection a full sentence? y/n y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Including subsection title in searchable text, but still include it as a separate subsection...\n",
      "\n",
      "-----------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#get .xml files from path, convert pdf to TEIFILE object\n",
    "#get dictionary of keywords and sentences\n",
    "#create formatted lists for creation of dataframe\n",
    "\n",
    "\n",
    "files = glob.glob(path + 'xml_output/' + '*.xml')\n",
    "#files = glob.glob('/Users/corinnsmall/Documents/BPDCN/bpdcn_papers/example/output/*.xml')\n",
    "\n",
    "files_list = []\n",
    "\n",
    "\n",
    "for file in files:\n",
    "    #print(file)\n",
    "    file_list = []\n",
    "    f = TEIFile(file)\n",
    "    name = f.filename.split('/')[-1].split('.')[0]\n",
    "    f.text\n",
    "    print('-----------------------------------------------------------')\n",
    "    print('')\n",
    "    \n",
    "    \n",
    "    f.keytext(keylist)\n",
    "    \n",
    "    for k,v in f._keytext.items():\n",
    "        for d in v:\n",
    "            #print('subsection dict: ', d)\n",
    "            for i,j in d.items():\n",
    "                #print(i, '\\n', j)\n",
    "                \n",
    "                if len(j) == 0 and f._check == None:\n",
    "                    file_list = [name, f.filename, f.doi, 'NA', k, i, 'NA']\n",
    "\n",
    "                elif f._check == None:\n",
    "                    file_list = [name, f.filename, f.doi, 'NA', k, i, j]   \n",
    "\n",
    "                else:\n",
    "                    file_list = [name, f.filename, f.doi, f._check, k, i, j]\n",
    "\n",
    "                files_list.append(file_list)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Ahoy!\n"
     ]
    }
   ],
   "source": [
    "#create dataframe and write to excel file\n",
    "\n",
    "df = pd.DataFrame(files_list, columns = ['pdf_name','pdf_location','doi', 'check_paper_content','keywords', 'section', 'text'])\n",
    "                \n",
    "try:\n",
    "    with open(path + output + '.xlsx', 'wb') as out:\n",
    "        df.to_excel(out)\n",
    "        \n",
    "    if out.closed:\n",
    "        print('Data Ahoy!')\n",
    "\n",
    "except IOError:\n",
    "        print('I/O error')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python391jvsc74a57bd0aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49",
   "display_name": "Python 3.9.1 64-bit"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "metadata": {
   "interpreter": {
    "hash": "70b38d7a306a849643e446cd70466270a13445e5987dfa1344ef2b127438fa4d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}