cookies-updater.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aa7ee2ee-ef0c-4b3e-88a8-c93656a1e4a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-15T09:10:35.534617Z",
     "iopub.status.busy": "2022-06-15T09:10:35.534261Z",
     "iopub.status.idle": "2022-06-15T09:10:35.593691Z",
     "shell.execute_reply": "2022-06-15T09:10:35.592632Z",
     "shell.execute_reply.started": "2022-06-15T09:10:35.534579Z"
    }
   },
   "source": [
    "# Cookies Updater"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3e57ec2",
   "metadata": {},
   "source": [
    "This notebook get users cookies collected by the Chrome extension from Naas Event API. It will then update the cookies information as a secret in the user Naas Lab server. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0fecc79-b882-48a0-b058-0535e3f1acbd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-15T09:10:19.792708Z",
     "iopub.status.busy": "2022-06-15T09:10:19.792390Z",
     "iopub.status.idle": "2022-06-15T09:10:19.797174Z",
     "shell.execute_reply": "2022-06-15T09:10:19.795767Z",
     "shell.execute_reply.started": "2022-06-15T09:10:19.792680Z"
    },
    "tags": []
   },
   "source": [
    "# Input"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d82d4b5-3ed7-4e61-a7da-a6cdc1894203",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-15T09:13:26.007601Z",
     "iopub.status.busy": "2022-06-15T09:13:26.007262Z",
     "iopub.status.idle": "2022-06-15T09:13:26.013410Z",
     "shell.execute_reply": "2022-06-15T09:13:26.011186Z",
     "shell.execute_reply.started": "2022-06-15T09:13:26.007564Z"
    }
   },
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76753922-f9fc-4993-98de-f5a8819e9aa6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:43.332851Z",
     "iopub.status.busy": "2022-09-29T11:10:43.332345Z",
     "iopub.status.idle": "2022-09-29T11:10:45.559364Z",
     "shell.execute_reply": "2022-09-29T11:10:45.558741Z",
     "shell.execute_reply.started": "2022-09-29T11:10:43.332715Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    import boto3\n",
    "except:\n",
    "    !pip install boto3 --user\n",
    "    import boto3  \n",
    "import naas\n",
    "import pydash\n",
    "\n",
    "import escapism\n",
    "import string\n",
    "\n",
    "import json\n",
    "import base64\n",
    "import uuid\n",
    "import datetime\n",
    "\n",
    "from dataclasses import dataclass, field\n",
    "try:\n",
    "    from dataclasses_json import dataclass_json, config\n",
    "except:\n",
    "    !pip install --user dataclasses-json\n",
    "    from dataclasses_json import dataclass_json, config"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69743572-09c6-4ada-920c-c04654e580b6",
   "metadata": {},
   "source": [
    "## Define globals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "03a93ff1-6669-4a1b-ac95-40e6524bbcb7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:45.560696Z",
     "iopub.status.busy": "2022-09-29T11:10:45.560460Z",
     "iopub.status.idle": "2022-09-29T11:10:45.565340Z",
     "shell.execute_reply": "2022-09-29T11:10:45.564742Z",
     "shell.execute_reply.started": "2022-09-29T11:10:45.560666Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "SQS_QUEUE_URL = 'https://sqs.eu-west-3.amazonaws.com/903885477968/linkedin-cookies-production.fifo'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "187551b3-92d7-4a3f-9090-88f8168122d1",
   "metadata": {},
   "source": [
    "## Configure SQS Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7032b24c-bb5f-4a8a-b639-7f29e6401e38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:45.566900Z",
     "iopub.status.busy": "2022-09-29T11:10:45.566676Z",
     "iopub.status.idle": "2022-09-29T11:10:54.040778Z",
     "shell.execute_reply": "2022-09-29T11:10:54.040171Z",
     "shell.execute_reply.started": "2022-09-29T11:10:45.566870Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ftp/.local/lib/python3.9/site-packages/botocore/client.py:625: FutureWarning: The sqs client is currently using a deprecated endpoint: eu-west-3.queue.amazonaws.com. In the next minor version this will be moved to sqs.eu-west-3.amazonaws.com. See https://github.com/boto/botocore/issues/2705 for more details.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "sqs = boto3.client(\n",
    "    'sqs',\n",
    "    region_name=\"eu-west-3\",\n",
    "    aws_access_key_id=naas.secret.get('LINKEDIN_COOKIES_UPDATER_AWS_ACCESS_KEY_ID'),\n",
    "    aws_secret_access_key=naas.secret.get('LINKEDIN_COOKIES_UPDATER_AWS_SECRET_ACCESS_KEY')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6361fde4-63b4-4b29-abdc-b12866e73734",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-15T09:10:09.395346Z",
     "iopub.status.busy": "2022-06-15T09:10:09.394902Z",
     "iopub.status.idle": "2022-06-15T09:10:09.398738Z",
     "shell.execute_reply": "2022-06-15T09:10:09.397860Z",
     "shell.execute_reply.started": "2022-06-15T09:10:09.395316Z"
    }
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f3585d6-188a-4f1e-b2fa-7869b9325ef7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-16T08:00:18.743659Z",
     "iopub.status.busy": "2022-06-16T08:00:18.743405Z",
     "iopub.status.idle": "2022-06-16T08:00:18.747646Z",
     "shell.execute_reply": "2022-06-16T08:00:18.746594Z",
     "shell.execute_reply.started": "2022-06-16T08:00:18.743635Z"
    }
   },
   "source": [
    "## Create helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bff79a18-a91b-4faa-85e2-222d7296d112",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:54.043817Z",
     "iopub.status.busy": "2022-09-29T11:10:54.043641Z",
     "iopub.status.idle": "2022-09-29T11:10:54.052867Z",
     "shell.execute_reply": "2022-09-29T11:10:54.050705Z",
     "shell.execute_reply.started": "2022-09-29T11:10:54.043795Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def escape(s):\n",
    "    return escapism.escape(\n",
    "    s, \n",
    "    safe=set(string.ascii_letters + string.digits), \n",
    "    escape_char='-').lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7f1c911-0ff4-496b-ae1d-644ff01ba4d1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-16T08:00:30.302849Z",
     "iopub.status.busy": "2022-06-16T08:00:30.302554Z",
     "iopub.status.idle": "2022-06-16T08:00:30.306738Z",
     "shell.execute_reply": "2022-06-16T08:00:30.305398Z",
     "shell.execute_reply.started": "2022-06-16T08:00:30.302813Z"
    }
   },
   "source": [
    "## Create Helper class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "212fc06b-94be-4bcc-8f61-9e37e52ed0b9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:54.056410Z",
     "iopub.status.busy": "2022-09-29T11:10:54.053892Z",
     "iopub.status.idle": "2022-09-29T11:10:54.140859Z",
     "shell.execute_reply": "2022-09-29T11:10:54.140039Z",
     "shell.execute_reply.started": "2022-09-29T11:10:54.056386Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "@dataclass_json\n",
    "@dataclass\n",
    "class EventContent:\n",
    "    li_at: str\n",
    "    jsessionid: str\n",
    "\n",
    "@dataclass_json\n",
    "@dataclass\n",
    "class Event:\n",
    "    event_type: str\n",
    "    event_content: EventContent\n",
    "\n",
    "@dataclass_json\n",
    "@dataclass\n",
    "class SQSMessage:\n",
    "    version: int\n",
    "    id: str\n",
    "    detail_type: str = field(metadata=config(field_name=\"detail-type\"))\n",
    "    source: str\n",
    "    account: str\n",
    "    time: str\n",
    "    region: str\n",
    "    detail: Event"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "501031cf-85ae-44a6-96d2-7b8dd6097281",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-16T08:00:41.462932Z",
     "iopub.status.busy": "2022-06-16T08:00:41.462641Z",
     "iopub.status.idle": "2022-06-16T08:00:41.466762Z",
     "shell.execute_reply": "2022-06-16T08:00:41.465771Z",
     "shell.execute_reply.started": "2022-06-16T08:00:41.462905Z"
    }
   },
   "source": [
    "## Create SQS functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7a46b0cd-dac4-4829-b615-0432388fc6a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:54.142568Z",
     "iopub.status.busy": "2022-09-29T11:10:54.142090Z",
     "iopub.status.idle": "2022-09-29T11:10:54.228706Z",
     "shell.execute_reply": "2022-09-29T11:10:54.227924Z",
     "shell.execute_reply.started": "2022-09-29T11:10:54.142533Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_sqs_messages(max_number_of_messages:int=1):\n",
    "    response = sqs.receive_message(\n",
    "        AttributeNames=['All'],\n",
    "        QueueUrl=SQS_QUEUE_URL,\n",
    "        WaitTimeSeconds=3,\n",
    "        MaxNumberOfMessages=max_number_of_messages\n",
    "    )\n",
    "    return response['Messages'] if 'Messages' in response else []\n",
    "\n",
    "def delete_sqs_message(receipt_handle: str):\n",
    "    return sqs.delete_message(\n",
    "        QueueUrl=SQS_QUEUE_URL,\n",
    "        ReceiptHandle=receipt_handle\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a4eb474-351e-4a8e-b263-e084e12b046b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-16T08:03:48.062288Z",
     "iopub.status.busy": "2022-06-16T08:03:48.062013Z",
     "iopub.status.idle": "2022-06-16T08:03:48.065883Z",
     "shell.execute_reply": "2022-06-16T08:03:48.064906Z",
     "shell.execute_reply.started": "2022-06-16T08:03:48.062260Z"
    }
   },
   "source": [
    "## Read, Encode and Decode secrets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "217e14af-9864-4b78-9279-4fa2f2ba1ad1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:54.230262Z",
     "iopub.status.busy": "2022-09-29T11:10:54.229834Z",
     "iopub.status.idle": "2022-09-29T11:10:54.313900Z",
     "shell.execute_reply": "2022-09-29T11:10:54.313148Z",
     "shell.execute_reply.started": "2022-09-29T11:10:54.230230Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def decode(secret_base64):\n",
    "    secret = base64.b64decode(secret_base64)\n",
    "    secret_decoded = secret.decode(\"ascii\")\n",
    "    return secret_decoded\n",
    "\n",
    "def encode(text):\n",
    "    message_bytes = text.encode(\"ascii\")\n",
    "    base64_bytes = base64.b64encode(message_bytes)\n",
    "    secret_base64 = base64_bytes.decode(\"ascii\")\n",
    "    return secret_base64\n",
    "\n",
    "def update_secret(target_file, key, value):\n",
    "    content = None\n",
    "    try:\n",
    "        with open(target_file, 'r') as f:\n",
    "            content = json.load(f)\n",
    "            f.close()\n",
    "    except:\n",
    "        content = []\n",
    "    \n",
    "    \n",
    "    r = pydash.find(content, lambda x: x['name'] == key)\n",
    "    now = datetime.datetime.now()\n",
    "    dt_string = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    if r != None:\n",
    "        r['lastUpdate'] = dt_string\n",
    "        r['secret'] = encode(value)\n",
    "    else:\n",
    "        content.append({\n",
    "            \"id\": str(uuid.uuid4()),\n",
    "            \"name\": key,\n",
    "            \"secret\": encode(value),\n",
    "            \"lastUpdate\": dt_string,\n",
    "        })\n",
    "\n",
    "    with open(target_file, 'w+') as wfile:\n",
    "        wfile.write(\n",
    "            json.dumps(content, sort_keys=True, indent=4).replace(\"NaN\", \"null\")\n",
    "        )\n",
    "        wfile.close()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cb4cd5a-16aa-4494-92c2-7b913a5625c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-16T08:00:52.138605Z",
     "iopub.status.busy": "2022-06-16T08:00:52.138317Z",
     "iopub.status.idle": "2022-06-16T08:00:52.142418Z",
     "shell.execute_reply": "2022-06-16T08:00:52.141424Z",
     "shell.execute_reply.started": "2022-06-16T08:00:52.138579Z"
    }
   },
   "source": [
    "## Handle an SQS message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1ab7de4b-d5aa-4e7d-ad39-b4cbe3354768",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:54.317701Z",
     "iopub.status.busy": "2022-09-29T11:10:54.317490Z",
     "iopub.status.idle": "2022-09-29T11:10:54.409282Z",
     "shell.execute_reply": "2022-09-29T11:10:54.408697Z",
     "shell.execute_reply.started": "2022-09-29T11:10:54.317675Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def handle_sqs_message(message):\n",
    "    # Get message  content\n",
    "    message_content = SQSMessage.from_json(message['Body'])\n",
    "    \n",
    "    # Get message type\n",
    "    message_type = message_content.detail_type\n",
    "    \n",
    "    # Make sure we have the right type\n",
    "    if message_type != 'linkedin.cookies':\n",
    "        print(message_content)\n",
    "        raise Exception('This message should not be in this SQS queue.')\n",
    "        \n",
    "    # Get the user to which this message belongs to.\n",
    "    user = message_content.source\n",
    "    \n",
    "    print(f'💡 Handling cookies for user {user}')\n",
    "    \n",
    "    # Get linkedin cookies.\n",
    "    li_at = message_content.detail.event_content.li_at\n",
    "    jsessionid = message_content.detail.event_content.jsessionid.replace('\"', '')\n",
    "    \n",
    "    print(f'LINKEDIN_LI_AT = {li_at}')\n",
    "    print(f'LINKEDIN_JSESSIONID = {jsessionid}')\n",
    "    \n",
    "    # Compute path to user's secrets file\n",
    "    user_secrets_path = f'/home/ftp/efs/dev/ftpusers/{escape(user)}/.naas/secrets.json'\n",
    "    \n",
    "    # Update secrets\n",
    "    update_secret(user_secrets_path, 'LINKEDIN_LI_AT', li_at)\n",
    "    update_secret(user_secrets_path, 'LINKEDIN_JSESSIONID', jsessionid)\n",
    "    \n",
    "    # Delete SQS message\n",
    "    delete_sqs_message(message['ReceiptHandle'])\n",
    "    \n",
    "    return user\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bf6e979-d26a-4fa1-adf1-13ac7074498b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-16T08:15:32.891226Z",
     "iopub.status.busy": "2022-06-16T08:15:32.890921Z",
     "iopub.status.idle": "2022-06-16T08:15:32.895824Z",
     "shell.execute_reply": "2022-06-16T08:15:32.894802Z",
     "shell.execute_reply.started": "2022-06-16T08:15:32.891191Z"
    },
    "tags": []
   },
   "source": [
    "## Handle all messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d80c5dbc-cbf2-46a6-9817-801547a8e301",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:54.412593Z",
     "iopub.status.busy": "2022-09-29T11:10:54.412392Z",
     "iopub.status.idle": "2022-09-29T11:10:54.496559Z",
     "shell.execute_reply": "2022-09-29T11:10:54.495960Z",
     "shell.execute_reply.started": "2022-09-29T11:10:54.412571Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def handle_all_messages():\n",
    "    while True:\n",
    "        messages = get_sqs_messages()\n",
    "        if len(messages) == 0:\n",
    "            break\n",
    "        for message in messages:\n",
    "            handle_sqs_message(message)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ade3ed37-bd70-472a-8c27-01f6943325f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-15T09:10:15.646647Z",
     "iopub.status.busy": "2022-06-15T09:10:15.646338Z",
     "iopub.status.idle": "2022-06-15T09:10:15.650416Z",
     "shell.execute_reply": "2022-06-15T09:10:15.649476Z",
     "shell.execute_reply.started": "2022-06-15T09:10:15.646609Z"
    }
   },
   "source": [
    "# Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "744bbdba-318f-4bfe-ba5f-88d33f3aa472",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:54.497821Z",
     "iopub.status.busy": "2022-09-29T11:10:54.497549Z",
     "iopub.status.idle": "2022-09-29T11:10:57.647950Z",
     "shell.execute_reply": "2022-09-29T11:10:57.647273Z",
     "shell.execute_reply.started": "2022-09-29T11:10:54.497789Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "handle_all_messages()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ee10d323-86e4-434d-ae13-dbb41197cea0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-29T11:10:57.649213Z",
     "iopub.status.busy": "2022-09-29T11:10:57.648988Z",
     "iopub.status.idle": "2022-09-29T11:10:58.020456Z",
     "shell.execute_reply": "2022-09-29T11:10:58.019652Z",
     "shell.execute_reply.started": "2022-09-29T11:10:57.649184Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": "\n            alert('Warning you just scheduled a notebook to run every minute!');\n            ",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "👌 Well done! Your Notebook has been sent to production.\n",
      "\n",
      "⏰ It will be scheduled \"Every minute of every day\" (more on the syntax on https://crontab.guru/).\n",
      "\n",
      "Ps: to remove the \"Scheduler\", just replace .add by .delete\n"
     ]
    }
   ],
   "source": [
    "naas.scheduler.add(cron=\"* * * * *\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4061bfc-bd60-4d82-a3fc-a95056dc6a01",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}