Anything-V3-Mirroring.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Linaqruf/sd-notebook-collection/blob/main/Anything-V3-Mirroring.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# A Notebook to Mirror Anything V3 to Your HuggingFace Repository\n",
        "### In case I make the repo private, you can use this notebook to mirror the model `(Full, fp32, fp16)` and upload/backup it to your Huggingface Repository. As long as I'm not resetting my `download` token. VAE in all model already replaced with the better one."
      ],
      "metadata": {
        "id": "yehBslknsPvU"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "hg27LwQuvvXK"
      },
      "outputs": [],
      "source": [
        "#@title ## Install Dependencies\n",
        "import os\n",
        "\n",
        "%cd /content/\n",
        "\n",
        "# Check if the directory already exists\n",
        "if os.path.isdir('/content/sd-notebook-collection'):\n",
        "  %cd /content/sd-notebook-collection\n",
        "  print(\"This folder already exists, will do a !git pull instead\\n\")\n",
        "  !git pull\n",
        "else:\n",
        "  !git clone https://github.com/Linaqruf/sd-notebook-collection\n",
        "\n",
        "%cd /content/sd-notebook-collection\n",
        "\n",
        "#@markdown This will install required Python packages\n",
        "!pip -qqqq install --upgrade -r requirements.txt\n",
        "!apt -qqqq install liblz4-tool aria2\n",
        "!pip -qqqq install huggingface"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Mirror Model\n",
        "%cd /content/\n",
        "from IPython.utils import capture\n",
        "\n",
        "models_path = \"/content/models\"\n",
        "\n",
        "if not os.path.exists(models_path):\n",
        "  print(f\"Creating directory at {models_path}\")\n",
        "  os.makedirs(models_path)\n",
        "\n",
        "#@markdown Which model to mirror?\n",
        "Anything_V_3_0_Full = True #@param {type:\"boolean\"}\n",
        "Anything_V_3_0_Pruned = True #@param {type:\"boolean\"}\n",
        "Anything_V_3_0_Pruned_fp16 = True #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown Do you want to upload diffusers format?\n",
        "diffusers_format = True #@param {type:\"boolean\"}\n",
        "\n",
        "full_model_url = \"https://huggingface.co/Linaqruf/personal-backup/resolve/main/vae-swapped-model/anything-v3-full.safetensors\"\n",
        "pruned_model_url = \"https://huggingface.co/Linaqruf/personal-backup/resolve/main/vae-swapped-model/anything-v3-fp32-pruned.safetensors\"\n",
        "fp16_model_url = \"https://huggingface.co/Linaqruf/personal-backup/resolve/main/vae-swapped-model/anything-v3-fp16-pruned.safetensors\"\n",
        "\n",
        "full_model_path = os.path.join(models_path, os.path.basename(full_model_url))\n",
        "pruned_model_path = os.path.join(models_path, os.path.basename(pruned_model_url))\n",
        "fp16_model_path = os.path.join(models_path, os.path.basename(fp16_model_url))\n",
        "\n",
        "base_name = os.path.splitext(os.path.basename(pruned_model_path))[0]\n",
        "diffusers_format = os.path.join(models_path, base_name)\n",
        "\n",
        "def download(url):\n",
        "  base_name= os.path.basename(url)\n",
        "  hf_token = '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' \n",
        "  user_header = f\"\\\"Authorization: Bearer {hf_token}\\\"\"\n",
        "  !aria2c --console-log-level=error --summary-interval=10 --header={user_header} -c -x 16 -k 1M -s 16 -d models -o {base_name} \"{url}\"\n",
        "\n",
        "def download_checkpoint():\n",
        "  if Anything_V_3_0_Full:\n",
        "      print(f\"Downloading full model from {full_model_url}\")\n",
        "      with capture.capture_output() as cap:\n",
        "        download(full_model_url)\n",
        "      print(\"Download completed!\")\n",
        "  if Anything_V_3_0_Pruned:\n",
        "      print(f\"Downloading pruned model from {pruned_model_url}\")\n",
        "      with capture.capture_output() as cap:\n",
        "        download(pruned_model_url)\n",
        "      print(\"Download completed!\")\n",
        "  if Anything_V_3_0_Pruned_fp16:\n",
        "      print(f\"Downloading pruned fp16 model from {fp16_model_url}\")\n",
        "      with capture.capture_output() as cap:\n",
        "        download(fp16_model_url)\n",
        "      print(\"Download completed!\")\n",
        "\n",
        "download_checkpoint()\n",
        "\n",
        "if diffusers_format:\n",
        "  %cd /content/sd-notebook-collection/script/\n",
        "  print(f\"Converting diffusers format for {diffusers_format}\")\n",
        "\n",
        "  reference_model = \"runwayml/stable-diffusion-v1-5\"\n",
        "\n",
        "  #@markdown Additional file for diffusers\n",
        "  feature_extractor = True #@param {'type': 'boolean'}\n",
        "  safety_checker = True #@param {'type': 'boolean'}\n",
        "\n",
        "  print(f'feature_extractor: {feature_extractor} safety_checker: {safety_checker}')\n",
        "  print(f'Please wait...')\n",
        "  with capture.capture_output() as cap: \n",
        "    !python convert_diffusers20_original_sd.py \\\n",
        "      \"{pruned_model_path}\" \\\n",
        "      \"{diffusers_format}\" \\\n",
        "      --v1 \\\n",
        "      --reference_model {reference_model} \n",
        "\n",
        "  url1 = \"https://huggingface.co/CompVis/stable-diffusion-safety-checker/resolve/main/preprocessor_config.json\"\n",
        "  url2 = \"https://huggingface.co/CompVis/stable-diffusion-safety-checker/resolve/main/config.json\"\n",
        "  url3 = \"https://huggingface.co/CompVis/stable-diffusion-safety-checker/resolve/main/pytorch_model.bin\"\n",
        "\n",
        "  if feature_extractor == True:\n",
        "    if not os.path.exists(str(diffusers_format)+'/feature_extractor'):\n",
        "      os.makedirs(str(diffusers_format)+'/feature_extractor')\n",
        "    \n",
        "    with capture.capture_output() as cap:  \n",
        "      !aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d '{diffusers_format}/feature_extractor' -Z {url1}\n",
        "\n",
        "  if safety_checker == True:\n",
        "    if not os.path.exists(str(diffusers_format)+'/safety_checker'):\n",
        "      os.makedirs(str(diffusers_format)+'/safety_checker')\n",
        "    \n",
        "    with capture.capture_output() as cap:    \n",
        "      !aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d '{diffusers_format}/safety_checker' -Z {url2}\n",
        "      !aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d '{diffusers_format}/safety_checker' -Z {url3}\n",
        "  \n",
        "  print(\"Conversion successful\")"
      ],
      "metadata": {
        "cellView": "form",
        "id": "n0R9Hb04DRud"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Upload Model\n",
        "#@markdown ### Login to Huggingface hub\n",
        "from huggingface_hub import login\n",
        "from huggingface_hub import HfApi\n",
        "from huggingface_hub.utils import validate_repo_id, HfHubHTTPError\n",
        "from pathlib import Path\n",
        "import shutil\n",
        "\n",
        "%cd /content/\n",
        "api = HfApi()\n",
        "\n",
        "#@markdown You can get your huggingface token from [this link](https://huggingface.co/settings/tokens), then `create new token` or copy available token with the `Write` role.\n",
        "\n",
        "write_token = \"your-write-token\" #@param {type:\"string\"}\n",
        "login(write_token, add_to_git_credential=True)\n",
        "\n",
        "user = api.whoami(write_token)\n",
        "\n",
        "#@markdown ### Define your Huggingface Repo\n",
        "#@markdown If your model repo didn't exist, it will automatically create your repo.\n",
        "model_name = \"anything-v3.0-mirror\" #@param{type:\"string\"}\n",
        "make_this_model_private = True #@param{type:\"boolean\"}\n",
        "\n",
        "model_repo = user['name']+\"/\"+model_name.strip()\n",
        "validate_repo_id(model_repo)\n",
        "\n",
        "if make_this_model_private:\n",
        "  private_repo = True\n",
        "else:\n",
        "  private_repo = False\n",
        "\n",
        "try:\n",
        "  api.create_repo(repo_id=model_repo, \n",
        "                  private=private_repo)\n",
        "  print(\"Model Repo didn't exists, creating repo\")\n",
        "  print(\"Model Repo: \",model_repo,\"created!\\n\")\n",
        "\n",
        "except HfHubHTTPError as e:\n",
        "  print(f\"Model Repo: {model_repo} exists, skipping create repo\\n\")\n",
        "\n",
        "def upload_model(model_path, is_diffusers: bool):\n",
        "  path_obj = Path(model_path)\n",
        "  trained_model = path_obj.parts[-1]\n",
        "  commit_message = f\"feat: upload {trained_model}\" \n",
        "  \n",
        "  if is_diffusers == True:\n",
        "    print(f\"Uploading diffusers model to https://huggingface.co/\"+model_repo)\n",
        "    print(f\"Please wait...\")\n",
        "\n",
        "    api.upload_folder(\n",
        "        folder_path=model_path,\n",
        "        repo_id=model_repo,\n",
        "        commit_message=commit_message,\n",
        "        ignore_patterns=\".ipynb_checkpoints\"\n",
        "    )\n",
        "    print(f\"Upload success, located at https://huggingface.co/\"+model_repo+\"/tree/main\\n\")\n",
        "  else:\n",
        "    print(f\"Uploading {trained_model} to https://huggingface.co/\"+model_repo)\n",
        "    print(f\"Please wait...\")\n",
        "    api.upload_file(\n",
        "        path_or_fileobj=model_path,\n",
        "        path_in_repo=trained_model,\n",
        "        repo_id=model_repo,\n",
        "        commit_message=commit_message,\n",
        "        )  \n",
        "    print(f\"Upload success, located at https://huggingface.co/\"+model_repo+\"/blob/main/\"+trained_model+\"\\n\")\n",
        "\n",
        "def upload():\n",
        "  if Anything_V_3_0_Pruned:\n",
        "    upload_model(pruned_model_path, False)\n",
        "  if Anything_V_3_0_Pruned_fp16:\n",
        "    upload_model(fp16_model_path, False)\n",
        "  if diffusers_format:\n",
        "    upload_model(diffusers_format, True)\n",
        "  if Anything_V_3_0_Full:\n",
        "    upload_model(full_model_path, False)\n",
        "\n",
        "upload()\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "jzOIuaFHF3V9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Commit Model\n",
        "\n",
        "import shutil\n",
        "from pathlib import Path\n",
        "%cd /content/\n",
        "\n",
        "!git lfs install --skip-smudge\n",
        "!export GIT_LFS_SKIP_SMUDGE=1\n",
        "!git clone https://huggingface.co/{model_repo} /content/{model_name}\n",
        "\n",
        "path_obj = Path(full_model_path)\n",
        "full_basename = path_obj.parts[-1]\n",
        "\n",
        "#@markdown Unfortunately you can't upload 7G model to huggingface with `hf_hub` using Colab Free, because it will crash the colab and automatically restart the runtime. So we will use git commit as alternative.\n",
        "repo = f\"/content/{model_name}\"\n",
        "# shutil.move(full_model_path, repo)\n",
        "#@markdown Set **git commit identity**\n",
        "email = \"your-email-here\" #@param {'type': 'string'}\n",
        "name = \"your-username-here\" #@param {'type': 'string'}\n",
        "#@markdown Set **commit message**\n",
        "commit_m = f\"feat: upload {full_basename}\"\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "1fxyjIkGULyU"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}