simstring_phon.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.6"
    },
    "colab": {
      "name": "simstring_phon.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/Text_Mining_Course/blob/master/simstring_phon.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "unZTKx1pCZoY"
      },
      "source": [
        "# Phonetic mapping of names\n",
        "\n",
        "* Captures regularities in name transliterations\n",
        "* Map name to a representation which is (hopefully) common to all its different transliterations\n",
        "* E.g. for Arabic names, you might want to drop vowels\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zftmHuiGCZoh",
        "outputId": "e630dc94-c94c-49bd-c4a0-19b7bb5463cf"
      },
      "source": [
        "!pip install abydos\n",
        "!pip install http://dl.turkunlp.org/textual-data-analysis-course-data/simstring-1.1-cp37-cp37m-linux_x86_64.whl"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting abydos\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7f/a5/ca258a571997be1c9483d6075bbc1b9487ae80f3bb3bf1f60db0b29f5aa6/abydos-0.5.0-py2.py3-none-any.whl (886kB)\n",
            "\r\u001b[K     |▍                               | 10kB 11.3MB/s eta 0:00:01\r\u001b[K     |▊                               | 20kB 8.9MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 7.2MB/s eta 0:00:01\r\u001b[K     |█▌                              | 40kB 7.3MB/s eta 0:00:01\r\u001b[K     |█▉                              | 51kB 4.5MB/s eta 0:00:01\r\u001b[K     |██▏                             | 61kB 5.1MB/s eta 0:00:01\r\u001b[K     |██▋                             | 71kB 5.0MB/s eta 0:00:01\r\u001b[K     |███                             | 81kB 5.3MB/s eta 0:00:01\r\u001b[K     |███▎                            | 92kB 5.2MB/s eta 0:00:01\r\u001b[K     |███▊                            | 102kB 5.5MB/s eta 0:00:01\r\u001b[K     |████                            | 112kB 5.5MB/s eta 0:00:01\r\u001b[K     |████▍                           | 122kB 5.5MB/s eta 0:00:01\r\u001b[K     |████▉                           | 133kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 143kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 153kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████                          | 163kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 174kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 184kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████                         | 194kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 204kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 215kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 225kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 235kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 245kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 256kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 266kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████                      | 276kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 286kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 296kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████                     | 307kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 317kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 327kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 337kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 348kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 358kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 368kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 378kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 389kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 399kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 409kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 419kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 430kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████                | 440kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 450kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 460kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 471kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 481kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 491kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 501kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 512kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 522kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 532kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 542kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 552kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 563kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 573kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 583kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 593kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 604kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 614kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 624kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 634kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 645kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 655kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 665kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 675kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 686kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 696kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 706kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 716kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 727kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 737kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 747kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 757kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 768kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 778kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 788kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 798kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 808kB 5.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 819kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 829kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 839kB 5.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 849kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 860kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 870kB 5.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 880kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 890kB 5.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from abydos) (1.19.5)\n",
            "Collecting deprecation\n",
            "  Downloading https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from deprecation->abydos) (20.9)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->deprecation->abydos) (2.4.7)\n",
            "Installing collected packages: deprecation, abydos\n",
            "Successfully installed abydos-0.5.0 deprecation-2.1.0\n",
            "Collecting simstring==1.1\n",
            "\u001b[?25l  Downloading http://dl.turkunlp.org/textual-data-analysis-course-data/simstring-1.1-cp37-cp37m-linux_x86_64.whl (893kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 273kB/s \n",
            "\u001b[?25hInstalling collected packages: simstring\n",
            "Successfully installed simstring-1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oYLQDs3vCZoj"
      },
      "source": [
        "import simstring\n",
        "import abydos"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UltiQv0JCZok",
        "outputId": "3240b6ae-f3bd-4857-8877-50b6b9dc355d"
      },
      "source": [
        "import abydos.phonetic\n",
        "encoder=abydos.phonetic.Metaphone()\n",
        "print(encoder.encode(\"Daniil Shafran\"))\n",
        "print(encoder.encode(\"Daniel Shafran\"))\n",
        "print(encoder.encode(\"Taniel Shafran\"))\n",
        "print(encoder.encode(\"Tamiel Shafran\"))\n",
        "\n",
        "\n",
        "print(encoder.encode(\"Evgeni Mravinsky\"))\n",
        "print(encoder.encode(\"Yevgeny Mravinsky\"))\n",
        "print(encoder.encode(\"Yevgeny Mriavinsky\"))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TNLXFRN\n",
            "TNLXFRN\n",
            "TNLXFRN\n",
            "TMLXFRN\n",
            "EFJNMRFNSK\n",
            "YFJNMRFNSK\n",
            "YFJNMRFNSK\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F4jyHpN3EZP7",
        "outputId": "783409b0-2608-4877-d4f3-8c96172ded36"
      },
      "source": [
        "!wget -nc http://dl.turkunlp.org/textual-data-analysis-course-data/wikidata.fi.bz2"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-03-10 17:13:27--  http://dl.turkunlp.org/textual-data-analysis-course-data/wikidata.fi.bz2\n",
            "Resolving dl.turkunlp.org (dl.turkunlp.org)... 195.148.30.23\n",
            "Connecting to dl.turkunlp.org (dl.turkunlp.org)|195.148.30.23|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 56380139 (54M) [application/octet-stream]\n",
            "Saving to: ‘wikidata.fi.bz2’\n",
            "\n",
            "wikidata.fi.bz2     100%[===================>]  53.77M  9.54MB/s    in 5.6s    \n",
            "\n",
            "2021-03-10 17:13:33 (9.54 MB/s) - ‘wikidata.fi.bz2’ saved [56380139/56380139]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C918iEzTCZol",
        "outputId": "228e3967-ee66-4084-9831-35e700898474"
      },
      "source": [
        "from tqdm import tqdm #progress-bar\n",
        "import pickle\n",
        "import os\n",
        "import bz2\n",
        "\n",
        "\n",
        "os.makedirs(\"wikidata_phon.db\",exist_ok=True)\n",
        "db=simstring.writer(\"wikidata_phon.db/wikidata_phon.db\")\n",
        "\n",
        "name_mapping={} #phonetic -> set(names)\n",
        "\n",
        "with bz2.open(\"wikidata.fi.bz2\",\"rt\") as f:\n",
        "    for line in tqdm(f):\n",
        "        line=line.strip()\n",
        "        # 4-col file with string, two urls, and official label\n",
        "        # let us index the strings\n",
        "        s,url1,url2,label=line.split(\"\\t\")\n",
        "        encoded_s=encoder.encode(s)\n",
        "        if encoded_s not in name_mapping: #a new string\n",
        "            db.insert(encoded_s)\n",
        "        name_mapping.setdefault(encoded_s,set()).add(s) #remember the string\n",
        "db.close()\n",
        "\n",
        "#store the name mapping \n",
        "with open(\"wikidata_phon.db/name_mapping.pickle\",\"wb\") as f:\n",
        "    pickle.dump(name_mapping,f)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3995911it [02:31, 26308.03it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RNvf4niSCZom"
      },
      "source": [
        "import pickle\n",
        "with open(\"wikidata_phon.db/name_mapping.pickle\",\"rb\") as f:\n",
        "    name_mapping=pickle.load(f)\n",
        "    \n",
        "db=simstring.reader(\"wikidata_phon.db/wikidata_phon.db\")\n",
        "db.metric=simstring.cosine"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9WSHM5SyJ1sK",
        "outputId": "dc271f7c-2881-4b2d-86cb-2c00b1ff1e33",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(len(name_mapping))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2271840\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P6GMdnzNCZon",
        "outputId": "5eaaa717-8f56-4836-cad9-3544bed57c0e"
      },
      "source": [
        "def retrieve_phon(s,db,threshold,encoder,name_mapping):\n",
        "    db.threshold=threshold\n",
        "    phon_hits=db.retrieve(encoder.encode(s))\n",
        "    return [name_mapping[ph] for ph in phon_hits]\n",
        "print(retrieve_phon(\"Tarja Halunen\",db,0.9,encoder,name_mapping)) #success\n",
        "print(retrieve_phon(\"Vlodymyr Puutin\",db,0.9,encoder,name_mapping)) #success\n",
        "print(retrieve_phon(\"Oleg Gordyievskyi\",db,0.6,encoder,name_mapping)) #fail\n",
        "print(retrieve_phon(\"Oleg Kordievski\",db,0.8,encoder,name_mapping)) #fail\n",
        "print(retrieve_phon(\"Oleg Gordyievskyi\",db,0.8,encoder,name_mapping)) #fail\n",
        "print(retrieve_phon(\"Oleg Gordievski\",db,0.8,encoder,name_mapping)) #success"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[{'Tarja Helanen', 'Tarja Halonen'}]\n",
            "[{'Vladimir Putin'}, {'Vladimir Potanin'}]\n",
            "[{'Toya fusca'}, {'Oli kord', 'Olokurto', 'Oelgard'}, {'Bradya fusca'}, {'Gordeyevsky'}]\n",
            "[{'Oleg Ogorodov'}]\n",
            "[]\n",
            "[{'Olga Rotševa'}, {'Gratofsky', 'Carte physique', 'Gardavská'}, {'Luokka:Eredivisie'}, {'Oleg Gordievski'}]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8p7qswiGCZop"
      },
      "source": [
        "## Possible improvements\n",
        "\n",
        "* These techniques compress the names by removing many vowels\n",
        "* 3-grams in simstring might be too long"
      ]
    }
  ]
}