en-tr-lstm-with-luong-attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "#import numpy as np # linear algebra\n",
    "#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install tensorflow==2.13.0 --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\n",
    "\n",
    "import re\n",
    "import io\n",
    "import unicodedata\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models, optimizers, losses\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = \"/kaggle/input/bilingual-sentence-pairs/tur.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = open(file_path, encoding=\"UTF-8\").read().strip().split(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(s):\n",
    "    turkish_chars = \"çğıöşüÇĞİÖŞÜ\"\n",
    "    english_equivalents = \"cgiosuCGIOSU\"\n",
    "    mapping = str.maketrans(turkish_chars, english_equivalents)\n",
    "    s = ''.join(c for c in unicodedata.normalize('NFD', s.lower().strip()) if unicodedata.category(c) != 'Mn')\n",
    "    s = re.sub(r'[çğıöşüÇĞİÖŞÜ]', lambda x: x.group(0).translate(mapping), s)\n",
    "    s = s.strip()\n",
    "    s = '<start> ' + s + ' <end>'\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_lang, input_lang = zip(*[[preprocess(word) for word in line.split(\"\\t\")[:-1]] for line in lines[:50000]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_tokenizer = Tokenizer(filters=\"\", oov_token=\"<unknown>\")\n",
    "input_tokenizer.fit_on_texts(input_lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_tensor = input_tokenizer.texts_to_sequences(input_lang)\n",
    "input_tensor = pad_sequences(input_tensor, padding=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_tokenizer = Tokenizer(filters=\"\", oov_token=\"<unknown>\")\n",
    "target_tokenizer.fit_on_texts(target_lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_tensor = target_tokenizer.texts_to_sequences(target_lang)\n",
    "target_tensor = pad_sequences(target_tensor, padding=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(input_tensor, target_tensor, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Input tensors:\", X_train.shape, X_test.shape)\n",
    "print(\"Target tensors:\", y_train.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(X_train)\n",
    "BATCH_SIZE = 64\n",
    "EMBEDDING_DIM = 256\n",
    "vocab_input_size = len(input_tokenizer.index_word) + 1\n",
    "vocab_target_size = len(target_tokenizer.index_word) + 1\n",
    "EPOCHS = 20\n",
    "STEPS_PER_EPOCH = len(X_train) // BATCH_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True).prefetch(1)\n",
    "valid_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(BATCH_SIZE, drop_remainder=True).prefetch(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_batch, target_batch = next(iter(train_dataset))\n",
    "print(\"Input Batch Shape:\", input_batch.shape)\n",
    "print(\"Target Batch Shape:\", target_batch.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(models.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, encoder_units, batch_size):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.encoder_units = encoder_units\n",
    "        self.batch_size = batch_size\n",
    "        self.embedding = layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = layers.LSTM(self.encoder_units, return_sequences=True, return_state=True)\n",
    "\n",
    "    def call(self, input_batch, state_h, state_c):\n",
    "        input_batch = self.embedding(input_batch)\n",
    "        output, state_h, state_c = self.lstm(input_batch, initial_state=[state_h, state_c])\n",
    "        return output, state_h, state_c\n",
    "\n",
    "    def initialize_state(self):\n",
    "        return [\n",
    "            tf.zeros((self.batch_size, self.encoder_units)),\n",
    "            tf.zeros((self.batch_size, self.encoder_units))\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder(vocab_input_size, EMBEDDING_DIM, 512, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[encoder_state_h, encoder_state_c] = encoder.initialize_state()\n",
    "encoder_output, encoder_state_h, encoder_state_c = encoder(input_batch, encoder_state_h, encoder_state_c)\n",
    "\n",
    "print(\"Encoder output shape:\", encoder_output.shape)\n",
    "print(\"Encoder hidden state h shape:\", encoder_state_h.shape)\n",
    "print(\"Encoder hidden state c shape:\", encoder_state_c.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Luong Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LuongAttention(layers.Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "    def call(self, decoder_state_h, decoder_state_c, encoder_output):\n",
    "        decoder_state = tf.add(decoder_state_h, decoder_state_c)[:, :, tf.newaxis]\n",
    "        score = layers.dot([encoder_output, decoder_state], axes=[2, 1])\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        context_vector = tf.reduce_sum(attention_weights * encoder_output, axis=1)\n",
    "        return context_vector, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_layer = LuongAttention()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder_state_h, decoder_state_c = encoder_state_h, encoder_state_c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(models.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, decoder_units, batch_size):\n",
    "        super().__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.embedding = layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = layers.LSTM(decoder_units, return_state=True, return_sequences=True)\n",
    "        self.fc = layers.Dense(vocab_size, activation=\"softmax\")\n",
    "        self.attention = LuongAttention()\n",
    "\n",
    "    def call(self, decoder_input, decoder_state_h, decoder_state_c, encoder_output):\n",
    "        context_vector, attention_weights = self.attention(decoder_state_h, decoder_state_c, encoder_output)\n",
    "        context_vector = context_vector[:, tf.newaxis, :]\n",
    "        x = self.embedding(decoder_input)\n",
    "        x = tf.concat([context_vector, x], axis=-1)\n",
    "\n",
    "        output, state_h, state_c = self.lstm(x)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        x = self.fc(output)\n",
    "        return x, state_h, state_c, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = Decoder(vocab_target_size, EMBEDDING_DIM, 512, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder_input = tf.random.uniform((BATCH_SIZE, 1))\n",
    "decoder_output, decoder_state_h, decoder_state_c, _ = decoder(decoder_input, decoder_state_h, decoder_state_c, encoder_output)\n",
    "\n",
    "print(\"Decoder output shape:\", decoder_output.shape)\n",
    "print(\"Decoder hidden state h shape:\", decoder_state_h.shape)\n",
    "print(\"Decoder hidden state c shape:\", decoder_state_c.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optimizers.Adam()\n",
    "loss_fn = losses.SparseCategoricalCrossentropy(reduction=\"none\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(input_batch, target_batch, encoder_state_h, encoder_state_c):\n",
    "    loss = 0\n",
    "    with tf.GradientTape() as tape:\n",
    "        encoder_output, encoder_state_h, encoder_state_c = encoder(input_batch, encoder_state_h, encoder_state_c)\n",
    "        decoder_state_h, decoder_state_c = encoder_state_h, encoder_state_c\n",
    "        decoder_input = tf.expand_dims([target_tokenizer.word_index[\"<start>\"]] * BATCH_SIZE, 1)\n",
    "        for target in range(1, target_batch.shape[1]):\n",
    "            predictions, decoder_state_h, decoder_state_c, _ = decoder(decoder_input, decoder_state_h, decoder_state_c, encoder_output)\n",
    "            mask = tf.cast(target_batch[:, target] != 0, dtype=predictions.dtype)\n",
    "            loss += tf.reduce_mean(loss_fn(target_batch[:, target], predictions) * mask)\n",
    "            decoder_input = tf.expand_dims(target_batch[:, target], 1)\n",
    "\n",
    "    batch_loss = loss / int(target_batch.shape[1])\n",
    "    optimizer.apply_gradients(zip(tape.gradient(loss, encoder.trainable_variables + decoder.trainable_variables), encoder.trainable_variables + decoder.trainable_variables))\n",
    "    return batch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_loss_arr = []\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    [encoder_state_h, encoder_state_c] = encoder.initialize_state()\n",
    "\n",
    "    total_batch_loss = 0\n",
    "    for (batch, (inp, targ)) in enumerate(train_dataset.take(STEPS_PER_EPOCH)):\n",
    "        batch_loss = train_step(inp, targ, encoder_state_h, encoder_state_c)\n",
    "        total_batch_loss += batch_loss \n",
    "\n",
    "    total_loss_arr.append(total_batch_loss / STEPS_PER_EPOCH)\n",
    "    print(f\"Epoch {epoch}, Loss: {total_batch_loss / STEPS_PER_EPOCH}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(total_loss_arr)\n",
    "plt.xlabel(\"Loss\")\n",
    "plt.title(\"Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(sentence, target_tensor, input_tensor):\n",
    "    sentence = preprocess(sentence)\n",
    "    inputs = [input_tokenizer.word_index[i] for i in sentence.split(\" \")]\n",
    "    inputs = pad_sequences([inputs], maxlen=input_tensor.shape[1], padding=\"post\")\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "\n",
    "    result = \"\"\n",
    "    [encoder_state_h, encoder_state_c] = [tf.zeros((1, 512)), tf.zeros((1, 512))]\n",
    "    encoder_output, encoder_state_h, encoder_state_c = encoder(inputs, encoder_state_h, encoder_state_c)\n",
    "    decoder_state_h, decoder_state_c = encoder_state_h, encoder_state_c\n",
    "    decoder_input = tf.expand_dims([target_tokenizer.word_index[\"<start>\"]], 0)\n",
    "    \n",
    "    for t in range(target_tensor.shape[1]):\n",
    "        predictions, decoder_state_h, decoder_state_c, attention_weights = decoder(decoder_input, decoder_state_h, decoder_state_c, encoder_output)\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "        result += target_tokenizer.index_word[predicted_id] + \" \"\n",
    "        \n",
    "        if target_tokenizer.index_word[predicted_id] == \"<end>\":\n",
    "            return result, sentence\n",
    "\n",
    "        decoder_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(sentence, ground_truth):\n",
    "    result, sentence = evaluate(sentence, target_tensor, input_tensor)\n",
    "\n",
    "    print(f'{\"Input:\":15s} {sentence}')\n",
    "    print(f'{\"Prediction:\":15s} {result}')\n",
    "    print(f'{\"Ground truth:\":15s} {ground_truth}') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for input_batch, target_batch in valid_dataset.take(10):\n",
    "    for inp, targ in zip(input_batch, target_batch):\n",
    "        sentence = input_tokenizer.sequences_to_texts([inp.numpy()])[0]\n",
    "        sentence = \" \".join([s for s in sentence.split(\" \") if s not in [\"<start>\", \"<end>\", \"<unknown>\"]])\n",
    "        ground_truth = target_tokenizer.sequences_to_texts([targ.numpy()])[0]\n",
    "        ground_truth = \" \".join([s for s in ground_truth.split(\" \") if s not in [\"<start>\", \"<end>\", \"<unknown>\"]])\n",
    "        translate(sentence, ground_truth)\n",
    "        print()\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [
    {
     "datasetId": 1118439,
     "sourceId": 1878727,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30699,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}