diff --git a/gptqmodel/models/base.py b/gptqmodel/models/base.py
index a2553c6c..a60c5c24 100644
--- a/gptqmodel/models/base.py
+++ b/gptqmodel/models/base.py
@@ -83,6 +83,8 @@ def __init__(
         qlinear_kernel: nn.Module = None,
     ):
         super().__init__()
+        logger.info(f"start base")
+
 
         self.model = model
         self.model_type = self.model.config.model_type
@@ -151,11 +153,24 @@ def _convert_tensor_to_list(tensor):
         return new_calibration_dataset_batched
 
     def quantize(
+            self,
+            calibration_dataset: List[Dict[str, Union[List[int], torch.LongTensor]]],
+            batch_size: int = 1,
+            calibration_enable_gpu_cache: bool = True,
+    ):
+        if isinstance(self.quantize_config, AutoRoundQuantizeConfig):
+            return self._quantize(calibration_dataset, batch_size, calibration_enable_gpu_cache)
+        else:
+            with torch.inference_mode():
+                return self._quantize(calibration_dataset, batch_size, calibration_enable_gpu_cache)
+
+    def _quantize(
         self,
         calibration_dataset: List[Dict[str, Union[List[int], torch.LongTensor]]],
         batch_size: int = 1,
         calibration_enable_gpu_cache: bool = True,
     ):
+        logger.info(f"start quant")
         if self.quantized:
             raise EnvironmentError("quantize() is called a model that is already quantized")
 
@@ -496,11 +511,13 @@ def tmp(_, inp, out):
                 [],
             )  # TODO: is it really OK to cache only the first positional argument?
             torch.cuda.empty_cache()
-
         logger.info(f"Quantization summary:\n{quant_log}")
         for module_log in quant_log:
             logger.info(module_log)
 
+        return quant_log, quantizers, force_layer_back_to_cpu, device_map, forward_pass_use_cache
+
+    def pack(self, quant_log, quantizers, force_layer_back_to_cpu, device_map, forward_pass_use_cache):
         self.qlinear_kernel = pack_model(
             model=self.model,
             quantizers=quantizers,
diff --git a/gptqmodel/utils/model.py b/gptqmodel/utils/model.py
index 0d7642f9..2faf3a99 100644
--- a/gptqmodel/utils/model.py
+++ b/gptqmodel/utils/model.py
@@ -15,6 +15,7 @@
 from tqdm import tqdm
 from transformers import AutoConfig, PretrainedConfig
 from transformers.utils.hub import cached_file
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from ..models._const import CPU, EXLLAMA_DEFAULT_MAX_INPUT_LENGTH, EXPERT_INDEX_PLACEHOLDER, SUPPORTED_MODELS
 from ..nn_modules.qlinear import BaseQuantLinear
@@ -32,7 +33,7 @@
 handler.setFormatter(formatter)
 logger.addHandler(handler)
 logger.setLevel(logging.INFO)
-
+logger.info(f"start base")
 
 def recurse_getattr(obj, attr: str):
     """
@@ -290,31 +291,41 @@ def pack_model(
     qlayers = find_layers(model, [QuantLinear])
 
     # Limit pack() thread usage to avoid auto-parallizataion regression
-    with tctl.threadpool_limits(limits=1):
-        pbar = tqdm(qlayers.keys(), leave=True)
-        for name in pbar:
-            pbar.set_description(f"Packing {name}")
-
-            quantizers[name], scale, zero, g_idx = quantizers[name]
-            # so far can only pack layer on CPU
-            layer_device = qlayers[name].device
-            qlayers[name].to(CPU)
-            layers[name], scale, zero, g_idx = (
-                layers[name].to(CPU),
-                scale.to(CPU),
-                zero.to(CPU),
-                g_idx.to(CPU),
-            )
-            if QuantLinear is MarlinQuantLinear:
-                qlayers[name].pack(layers[name], scale)
-            else:
-                qlayers[name].pack(layers[name], scale, zero, g_idx)
-            qlayers[name].to(layer_device)
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        # future_to_item = {executor.submit(pack_layer, quantizers[name], qlayers[name], layers[name], QuantLinear): name for name in qlayers.keys()}
+        # for future in tqdm(as_completed(future_to_item), total=len(qlayers.keys())):
+        #     item = future_to_item[future]
+        #     result = future.result()
+        #     print(f"Processed {item} to {result}")
+        futures = [(executor.submit(pack_layer, quantizers[name], qlayers[name], layers[name], QuantLinear), name) for name in qlayers.keys()]
+        for item in futures:
+            future, name = item
+            future.result()
+            print(f"{name} Processed")
 
     logger.info("Model packed.")
 
     return QuantLinear
 
+def pack_layer(quantizer, qlayer, layer, QuantLinear):
+    with tctl.threadpool_limits(limits=1):
+        _quantizer, scale, zero, g_idx = quantizer
+        # so far can only pack layer on CPU
+        layer_device = qlayer.device
+        qlayer.to(CPU)
+        layer, scale, zero, g_idx = (
+            layer.to(CPU),
+            scale.to(CPU),
+            zero.to(CPU),
+            g_idx.to(CPU),
+        )
+        if QuantLinear is MarlinQuantLinear:
+            qlayer.pack(layer, scale)
+        else:
+            qlayer.pack(layer, scale, zero, g_idx)
+        qlayer.to(layer_device)
+
+
 def verify_model_hash(file_path: str, verify_hash: str):
     if not isinstance(verify_hash, str):
         raise ValueError("model verify_hash must be a string")
diff --git a/multithread_pack.ipynb b/multithread_pack.ipynb
new file mode 100644
index 00000000..62ff5534
--- /dev/null
+++ b/multithread_pack.ipynb
@@ -0,0 +1,1616 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "de73aa28-0fcb-4f40-848f-29486f5a4638",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO - start base\n",
+      "INFO 2024-07-17 20:26:58.042 [61754-MainThread] (refresh_log@4181186596.py:039) init log\n",
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
+      "Token indices sequence length is longer than the specified maximum sequence length for this model (128838 > 32768). Running this sequence through the model will result in indexing errors\n",
+      "INFO 2024-07-17 20:26:59.867 [61754-MainThread] (<module>@4181186596.py:074) load data 300 examples 1.64\n"
+     ]
+    }
+   ],
+   "source": [
+    "#!/usr/bin/env python3\n",
+    "# -*- coding:utf-8 -*-\n",
+    "import os\n",
+    "# set env first then load other\n",
+    "os.environ['OMP_NUM_THREADS'] = '12'\n",
+    "os.environ['OPENBLAS_NUM_THREADS'] = '12'\n",
+    "os.environ['MKL_NUM_THREADS'] = '12'\n",
+    "os.environ['VECLIB_MAXIMUM_THREADS'] = '12'\n",
+    "os.environ['NUMEXPR_NUM_THREADS'] = '12'\n",
+    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1,2,5,7\"\n",
+    "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n",
+    "\n",
+    "import time\n",
+    "from transformers import AutoTokenizer\n",
+    "from transformers import AutoConfig, AutoModelForCausalLM, PreTrainedModel\n",
+    "from gptqmodel import GPTQModel, QuantizeConfig\n",
+    "import datasets\n",
+    "import accelerate\n",
+    "\n",
+    "import logging\n",
+    "def refresh_log():\n",
+    "    if logging.root.handlers:\n",
+    "        logging.info(f\"logging already init. reset all\")\n",
+    "        for handler in logging.root.handlers[:]:\n",
+    "            print(f\"{handler.name}\")\n",
+    "            logging.root.removeHandler(handler)\n",
+    "            handler.close()\n",
+    "\n",
+    "    \n",
+    "    logger_dict = logging.Logger.manager\n",
+    "    logger_dict.loggerDict = {}\n",
+    "\n",
+    "    \n",
+    "\n",
+    "    format = \"%(levelname)s %(asctime)s.%(msecs)03d [%(process)d-%(threadName)s] (%(funcName)s@%(filename)s:%(lineno)03d) %(message)s\"\n",
+    "    datefmt = \"%Y-%m-%d %H:%M:%S\"\n",
+    "    logging.basicConfig(format=format, datefmt=datefmt, level=logging.INFO, handlers=[logging.StreamHandler()])\n",
+    "    logging.info(f\"init log\")\n",
+    "\n",
+    "refresh_log()\n",
+    "\n",
+    "import builtins\n",
+    "def print(*args, **kwargs):\n",
+    "    sep = kwargs.get(\"sep\", \" \")  # default separator\n",
+    "    end = kwargs.get(\"end\", \"\")  # default end\n",
+    "    message = sep.join(map(str, args)) + end\n",
+    "    logging.info(message, stacklevel=3)\n",
+    "\n",
+    "builtins.print = print\n",
+    "\n",
+    "\n",
+    "# pretrained_model_dir = '/aigc-nas02/zpfcode/online_model/xdlovers_72b_0712'\n",
+    "pretrained_model_dir = '/aigc-nas02/hf_models/Qwen1.5-0.5B'\n",
+    "quantized_model_dir = f'{pretrained_model_dir}-gptq-int8'\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(pretrained_model_dir, use_fast=True)\n",
+    "\n",
+    "\n",
+    "quantize_config = QuantizeConfig(\n",
+    "    bits=8,  # quantize model to 4-bit\n",
+    "    group_size=128,  # it is recommended to set the value to 128\n",
+    "    desc_act=False,  # set to False can significantly speed up inference but the perplexity may slightly bad\n",
+    "    damp_percent=0.01,\n",
+    ")\n",
+    "\n",
+    "# max_memory={ 0:\"10GIB\", 1:\"64GIB\", 2:\"64GIB\", 3:\"64GIB\", \"cpu\":\"200GIB\"}\n",
+    "\n",
+    "# # device_map=\"balanced_low_0\",\n",
+    "# model = GPTQModel.from_pretrained(pretrained_model_dir, quantize_config, trust_remote_code=True,\n",
+    "#                                   max_memory=max_memory,\n",
+    "#                                      attn_implementation=\"flash_attention_2\") \n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0970ec88-bcf3-4307-b768-e4bf4e049d2c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "examples = []\n",
+    "ds_st = time.perf_counter()\n",
+    "pile_dataset: datasets.Dataset = datasets.load_dataset(r\"/aigc-nas02/cyj/pile-val-backup\", split='validation')\n",
+    "\n",
+    "def split_slow_tokenize(dataset: datasets.Dataset):\n",
+    "    dataset_10k = dataset[:300]\n",
+    "    examples = []\n",
+    "    for data in dataset_10k['text']:\n",
+    "        examples.append(tokenizer(data))\n",
+    "    return examples\n",
+    "\n",
+    "\n",
+    "examples = split_slow_tokenize(pile_dataset)\n",
+    "logging.info(f\"load data {len(examples)} examples {time.perf_counter() - ds_st:.2f}\")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "cbd75369-03aa-4790-a232-c0ec34d51c21",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO 2024-07-17 20:20:43.803 [12438-MainThread] (<module>@2049556245.py:001) start quantize\n",
+      "INFO - start quant\n",
+      "Quantizing self_attn.k_proj in layer 1 of 24:   0%|          | 0/24 [00:01<?, ?it/s]INFO - {'layer': 1, 'module': 'self_attn.k_proj', 'avg_loss': '0.0039', 'time': '0.2490'}\n",
+      "Quantizing self_attn.v_proj in layer 1 of 24:   0%|          | 0/24 [00:01<?, ?it/s]INFO - {'layer': 1, 'module': 'self_attn.v_proj', 'avg_loss': '0.0001', 'time': '0.2508'}\n",
+      "Quantizing self_attn.q_proj in layer 1 of 24:   0%|          | 0/24 [00:01<?, ?it/s]INFO - {'layer': 1, 'module': 'self_attn.q_proj', 'avg_loss': '0.0058', 'time': '0.2535'}\n",
+      "Quantizing self_attn.o_proj in layer 1 of 24:   0%|          | 0/24 [00:03<?, ?it/s]INFO - {'layer': 1, 'module': 'self_attn.o_proj', 'avg_loss': '0.0000', 'time': '0.2501'}\n",
+      "Quantizing mlp.up_proj in layer 1 of 24:   0%|          | 0/24 [00:04<?, ?it/s]     INFO - {'layer': 1, 'module': 'mlp.up_proj', 'avg_loss': '0.0183', 'time': '0.2542'}\n",
+      "Quantizing mlp.gate_proj in layer 1 of 24:   0%|          | 0/24 [00:04<?, ?it/s]INFO - {'layer': 1, 'module': 'mlp.gate_proj', 'avg_loss': '0.0249', 'time': '0.2524'}\n",
+      "Quantizing mlp.down_proj in layer 1 of 24:   0%|          | 0/24 [00:06<?, ?it/s]INFO - {'layer': 1, 'module': 'mlp.down_proj', 'avg_loss': '0.0009', 'time': '0.7110'}\n",
+      "Quantizing self_attn.k_proj in layer 2 of 24:   4%|▍         | 1/24 [00:09<03:17,  8.60s/it]INFO - {'layer': 2, 'module': 'self_attn.k_proj', 'avg_loss': '0.0487', 'time': '0.2542'}\n",
+      "Quantizing self_attn.v_proj in layer 2 of 24:   4%|▍         | 1/24 [00:10<03:17,  8.60s/it]INFO - {'layer': 2, 'module': 'self_attn.v_proj', 'avg_loss': '0.0056', 'time': '0.2556'}\n",
+      "Quantizing self_attn.q_proj in layer 2 of 24:   4%|▍         | 1/24 [00:10<03:17,  8.60s/it]INFO - {'layer': 2, 'module': 'self_attn.q_proj', 'avg_loss': '0.0434', 'time': '0.2489'}\n",
+      "Quantizing self_attn.o_proj in layer 2 of 24:   4%|▍         | 1/24 [00:11<03:17,  8.60s/it]INFO - {'layer': 2, 'module': 'self_attn.o_proj', 'avg_loss': '0.0001', 'time': '0.2569'}\n",
+      "Quantizing mlp.up_proj in layer 2 of 24:   4%|▍         | 1/24 [00:13<03:17,  8.60s/it]     INFO - {'layer': 2, 'module': 'mlp.up_proj', 'avg_loss': '0.0265', 'time': '0.2470'}\n",
+      "Quantizing mlp.gate_proj in layer 2 of 24:   4%|▍         | 1/24 [00:13<03:17,  8.60s/it]INFO - {'layer': 2, 'module': 'mlp.gate_proj', 'avg_loss': '0.0431', 'time': '0.2478'}\n",
+      "Quantizing mlp.down_proj in layer 2 of 24:   4%|▍         | 1/24 [00:15<03:17,  8.60s/it]INFO - {'layer': 2, 'module': 'mlp.down_proj', 'avg_loss': '0.0006', 'time': '0.6856'}\n",
+      "Quantizing self_attn.k_proj in layer 3 of 24:   8%|▊         | 2/24 [00:18<03:07,  8.53s/it]INFO - {'layer': 3, 'module': 'self_attn.k_proj', 'avg_loss': '0.0950', 'time': '0.2479'}\n",
+      "Quantizing self_attn.v_proj in layer 3 of 24:   8%|▊         | 2/24 [00:18<03:07,  8.53s/it]INFO - {'layer': 3, 'module': 'self_attn.v_proj', 'avg_loss': '0.0140', 'time': '0.2482'}\n",
+      "Quantizing self_attn.q_proj in layer 3 of 24:   8%|▊         | 2/24 [00:18<03:07,  8.53s/it]INFO - {'layer': 3, 'module': 'self_attn.q_proj', 'avg_loss': '0.0834', 'time': '0.2478'}\n",
+      "Quantizing self_attn.o_proj in layer 3 of 24:   8%|▊         | 2/24 [00:20<03:07,  8.53s/it]INFO - {'layer': 3, 'module': 'self_attn.o_proj', 'avg_loss': '0.0002', 'time': '0.2489'}\n",
+      "Quantizing mlp.up_proj in layer 3 of 24:   8%|▊         | 2/24 [00:21<03:07,  8.53s/it]     INFO - {'layer': 3, 'module': 'mlp.up_proj', 'avg_loss': '0.0338', 'time': '0.2457'}\n",
+      "Quantizing mlp.gate_proj in layer 3 of 24:   8%|▊         | 2/24 [00:21<03:07,  8.53s/it]INFO - {'layer': 3, 'module': 'mlp.gate_proj', 'avg_loss': '0.0550', 'time': '0.2464'}\n",
+      "Quantizing mlp.down_proj in layer 3 of 24:   8%|▊         | 2/24 [00:23<03:07,  8.53s/it]INFO - {'layer': 3, 'module': 'mlp.down_proj', 'avg_loss': '0.0009', 'time': '0.6789'}\n",
+      "Quantizing self_attn.k_proj in layer 4 of 24:  12%|█▎        | 3/24 [00:26<02:59,  8.55s/it]INFO - {'layer': 4, 'module': 'self_attn.k_proj', 'avg_loss': '0.0903', 'time': '0.2479'}\n",
+      "Quantizing self_attn.v_proj in layer 4 of 24:  12%|█▎        | 3/24 [00:27<02:59,  8.55s/it]INFO - {'layer': 4, 'module': 'self_attn.v_proj', 'avg_loss': '0.0177', 'time': '0.2490'}\n",
+      "Quantizing self_attn.q_proj in layer 4 of 24:  12%|█▎        | 3/24 [00:27<02:59,  8.55s/it]INFO - {'layer': 4, 'module': 'self_attn.q_proj', 'avg_loss': '0.0796', 'time': '0.2491'}\n",
+      "Quantizing self_attn.o_proj in layer 4 of 24:  12%|█▎        | 3/24 [00:28<02:59,  8.55s/it]INFO - {'layer': 4, 'module': 'self_attn.o_proj', 'avg_loss': '0.0003', 'time': '0.2473'}\n",
+      "Quantizing mlp.up_proj in layer 4 of 24:  12%|█▎        | 3/24 [00:30<02:59,  8.55s/it]     INFO - {'layer': 4, 'module': 'mlp.up_proj', 'avg_loss': '0.0439', 'time': '0.2482'}\n",
+      "Quantizing mlp.gate_proj in layer 4 of 24:  12%|█▎        | 3/24 [00:30<02:59,  8.55s/it]INFO - {'layer': 4, 'module': 'mlp.gate_proj', 'avg_loss': '0.0609', 'time': '0.2463'}\n",
+      "Quantizing mlp.down_proj in layer 4 of 24:  12%|█▎        | 3/24 [00:32<02:59,  8.55s/it]INFO - {'layer': 4, 'module': 'mlp.down_proj', 'avg_loss': '0.0013', 'time': '0.6797'}\n",
+      "Quantizing self_attn.k_proj in layer 5 of 24:  17%|█▋        | 4/24 [00:35<02:50,  8.53s/it]INFO - {'layer': 5, 'module': 'self_attn.k_proj', 'avg_loss': '0.0705', 'time': '0.2469'}\n",
+      "Quantizing self_attn.v_proj in layer 5 of 24:  17%|█▋        | 4/24 [00:35<02:50,  8.53s/it]INFO - {'layer': 5, 'module': 'self_attn.v_proj', 'avg_loss': '0.0124', 'time': '0.2492'}\n",
+      "Quantizing self_attn.q_proj in layer 5 of 24:  17%|█▋        | 4/24 [00:35<02:50,  8.53s/it]INFO - {'layer': 5, 'module': 'self_attn.q_proj', 'avg_loss': '0.0630', 'time': '0.2472'}\n",
+      "Quantizing self_attn.o_proj in layer 5 of 24:  17%|█▋        | 4/24 [00:37<02:50,  8.53s/it]INFO - {'layer': 5, 'module': 'self_attn.o_proj', 'avg_loss': '0.0004', 'time': '0.2458'}\n",
+      "Quantizing mlp.up_proj in layer 5 of 24:  17%|█▋        | 4/24 [00:38<02:50,  8.53s/it]     INFO - {'layer': 5, 'module': 'mlp.up_proj', 'avg_loss': '0.0506', 'time': '0.2469'}\n",
+      "Quantizing mlp.gate_proj in layer 5 of 24:  17%|█▋        | 4/24 [00:38<02:50,  8.53s/it]INFO - {'layer': 5, 'module': 'mlp.gate_proj', 'avg_loss': '0.0733', 'time': '0.2465'}\n",
+      "Quantizing mlp.down_proj in layer 5 of 24:  17%|█▋        | 4/24 [00:40<02:50,  8.53s/it]INFO - {'layer': 5, 'module': 'mlp.down_proj', 'avg_loss': '0.0014', 'time': '0.6803'}\n",
+      "Quantizing self_attn.k_proj in layer 6 of 24:  21%|██        | 5/24 [00:43<02:42,  8.53s/it]INFO - {'layer': 6, 'module': 'self_attn.k_proj', 'avg_loss': '0.0940', 'time': '0.2497'}\n",
+      "Quantizing self_attn.v_proj in layer 6 of 24:  21%|██        | 5/24 [00:44<02:42,  8.53s/it]INFO - {'layer': 6, 'module': 'self_attn.v_proj', 'avg_loss': '0.0180', 'time': '0.2521'}\n",
+      "Quantizing self_attn.q_proj in layer 6 of 24:  21%|██        | 5/24 [00:44<02:42,  8.53s/it]INFO - {'layer': 6, 'module': 'self_attn.q_proj', 'avg_loss': '0.0856', 'time': '0.2488'}\n",
+      "Quantizing self_attn.o_proj in layer 6 of 24:  21%|██        | 5/24 [00:45<02:42,  8.53s/it]INFO - {'layer': 6, 'module': 'self_attn.o_proj', 'avg_loss': '0.0007', 'time': '0.2459'}\n",
+      "Quantizing mlp.up_proj in layer 6 of 24:  21%|██        | 5/24 [00:47<02:42,  8.53s/it]     INFO - {'layer': 6, 'module': 'mlp.up_proj', 'avg_loss': '0.0614', 'time': '0.2470'}\n",
+      "Quantizing mlp.gate_proj in layer 6 of 24:  21%|██        | 5/24 [00:47<02:42,  8.53s/it]INFO - {'layer': 6, 'module': 'mlp.gate_proj', 'avg_loss': '0.0813', 'time': '0.2456'}\n",
+      "Quantizing mlp.down_proj in layer 6 of 24:  21%|██        | 5/24 [00:49<02:42,  8.53s/it]INFO - {'layer': 6, 'module': 'mlp.down_proj', 'avg_loss': '0.0075', 'time': '0.6766'}\n",
+      "Quantizing self_attn.k_proj in layer 7 of 24:  25%|██▌       | 6/24 [00:52<02:34,  8.57s/it]INFO - {'layer': 7, 'module': 'self_attn.k_proj', 'avg_loss': '0.0902', 'time': '0.2492'}\n",
+      "Quantizing self_attn.v_proj in layer 7 of 24:  25%|██▌       | 6/24 [00:52<02:34,  8.57s/it]INFO - {'layer': 7, 'module': 'self_attn.v_proj', 'avg_loss': '0.0270', 'time': '0.2521'}\n",
+      "Quantizing self_attn.q_proj in layer 7 of 24:  25%|██▌       | 6/24 [00:53<02:34,  8.57s/it]INFO - {'layer': 7, 'module': 'self_attn.q_proj', 'avg_loss': '0.0869', 'time': '0.2503'}\n",
+      "Quantizing self_attn.o_proj in layer 7 of 24:  25%|██▌       | 6/24 [00:54<02:34,  8.57s/it]INFO - {'layer': 7, 'module': 'self_attn.o_proj', 'avg_loss': '0.0008', 'time': '0.2488'}\n",
+      "Quantizing mlp.up_proj in layer 7 of 24:  25%|██▌       | 6/24 [00:55<02:34,  8.57s/it]     INFO - {'layer': 7, 'module': 'mlp.up_proj', 'avg_loss': '0.0667', 'time': '0.2496'}\n",
+      "Quantizing mlp.gate_proj in layer 7 of 24:  25%|██▌       | 6/24 [00:56<02:34,  8.57s/it]INFO - {'layer': 7, 'module': 'mlp.gate_proj', 'avg_loss': '0.0965', 'time': '0.2494'}\n",
+      "Quantizing mlp.down_proj in layer 7 of 24:  25%|██▌       | 6/24 [00:57<02:34,  8.57s/it]INFO - {'layer': 7, 'module': 'mlp.down_proj', 'avg_loss': '0.0022', 'time': '0.6853'}\n",
+      "Quantizing self_attn.k_proj in layer 8 of 24:  29%|██▉       | 7/24 [01:01<02:25,  8.58s/it]INFO - {'layer': 8, 'module': 'self_attn.k_proj', 'avg_loss': '0.0968', 'time': '0.2501'}\n",
+      "Quantizing self_attn.v_proj in layer 8 of 24:  29%|██▉       | 7/24 [01:01<02:25,  8.58s/it]INFO - {'layer': 8, 'module': 'self_attn.v_proj', 'avg_loss': '0.0316', 'time': '0.2510'}\n",
+      "Quantizing self_attn.q_proj in layer 8 of 24:  29%|██▉       | 7/24 [01:01<02:25,  8.58s/it]INFO - {'layer': 8, 'module': 'self_attn.q_proj', 'avg_loss': '0.0997', 'time': '0.2578'}\n",
+      "Quantizing self_attn.o_proj in layer 8 of 24:  29%|██▉       | 7/24 [01:02<02:25,  8.58s/it]INFO - {'layer': 8, 'module': 'self_attn.o_proj', 'avg_loss': '0.0010', 'time': '0.2455'}\n",
+      "Quantizing mlp.up_proj in layer 8 of 24:  29%|██▉       | 7/24 [01:04<02:25,  8.58s/it]     INFO - {'layer': 8, 'module': 'mlp.up_proj', 'avg_loss': '0.0680', 'time': '0.2488'}\n",
+      "Quantizing mlp.gate_proj in layer 8 of 24:  29%|██▉       | 7/24 [01:04<02:25,  8.58s/it]INFO - {'layer': 8, 'module': 'mlp.gate_proj', 'avg_loss': '0.0904', 'time': '0.2472'}\n",
+      "Quantizing mlp.down_proj in layer 8 of 24:  29%|██▉       | 7/24 [01:06<02:25,  8.58s/it]INFO - {'layer': 8, 'module': 'mlp.down_proj', 'avg_loss': '0.0024', 'time': '0.6778'}\n",
+      "Quantizing self_attn.k_proj in layer 9 of 24:  33%|███▎      | 8/24 [01:09<02:17,  8.59s/it]INFO - {'layer': 9, 'module': 'self_attn.k_proj', 'avg_loss': '0.0799', 'time': '0.2464'}\n",
+      "Quantizing self_attn.v_proj in layer 9 of 24:  33%|███▎      | 8/24 [01:10<02:17,  8.59s/it]INFO - {'layer': 9, 'module': 'self_attn.v_proj', 'avg_loss': '0.0242', 'time': '0.2489'}\n",
+      "Quantizing self_attn.q_proj in layer 9 of 24:  33%|███▎      | 8/24 [01:10<02:17,  8.59s/it]INFO - {'layer': 9, 'module': 'self_attn.q_proj', 'avg_loss': '0.0754', 'time': '0.2512'}\n",
+      "Quantizing self_attn.o_proj in layer 9 of 24:  33%|███▎      | 8/24 [01:11<02:17,  8.59s/it]INFO - {'layer': 9, 'module': 'self_attn.o_proj', 'avg_loss': '0.0012', 'time': '0.2452'}\n",
+      "Quantizing mlp.up_proj in layer 9 of 24:  33%|███▎      | 8/24 [01:12<02:17,  8.59s/it]     INFO - {'layer': 9, 'module': 'mlp.up_proj', 'avg_loss': '0.0668', 'time': '0.2462'}\n",
+      "Quantizing mlp.gate_proj in layer 9 of 24:  33%|███▎      | 8/24 [01:13<02:17,  8.59s/it]INFO - {'layer': 9, 'module': 'mlp.gate_proj', 'avg_loss': '0.0844', 'time': '0.2466'}\n",
+      "Quantizing mlp.down_proj in layer 9 of 24:  33%|███▎      | 8/24 [01:14<02:17,  8.59s/it]INFO - {'layer': 9, 'module': 'mlp.down_proj', 'avg_loss': '0.0025', 'time': '0.6810'}\n",
+      "Quantizing self_attn.k_proj in layer 10 of 24:  38%|███▊      | 9/24 [01:18<02:08,  8.55s/it]INFO - {'layer': 10, 'module': 'self_attn.k_proj', 'avg_loss': '0.0775', 'time': '0.2490'}\n",
+      "Quantizing self_attn.v_proj in layer 10 of 24:  38%|███▊      | 9/24 [01:18<02:08,  8.55s/it]INFO - {'layer': 10, 'module': 'self_attn.v_proj', 'avg_loss': '0.0229', 'time': '0.2503'}\n",
+      "Quantizing self_attn.q_proj in layer 10 of 24:  38%|███▊      | 9/24 [01:18<02:08,  8.55s/it]INFO - {'layer': 10, 'module': 'self_attn.q_proj', 'avg_loss': '0.0778', 'time': '0.2487'}\n",
+      "Quantizing self_attn.o_proj in layer 10 of 24:  38%|███▊      | 9/24 [01:20<02:08,  8.55s/it]INFO - {'layer': 10, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2569'}\n",
+      "Quantizing mlp.up_proj in layer 10 of 24:  38%|███▊      | 9/24 [01:21<02:08,  8.55s/it]     INFO - {'layer': 10, 'module': 'mlp.up_proj', 'avg_loss': '0.0604', 'time': '0.2461'}\n",
+      "Quantizing mlp.gate_proj in layer 10 of 24:  38%|███▊      | 9/24 [01:21<02:08,  8.55s/it]INFO - {'layer': 10, 'module': 'mlp.gate_proj', 'avg_loss': '0.0760', 'time': '0.2470'}\n",
+      "Quantizing mlp.down_proj in layer 10 of 24:  38%|███▊      | 9/24 [01:23<02:08,  8.55s/it]INFO - {'layer': 10, 'module': 'mlp.down_proj', 'avg_loss': '0.0021', 'time': '0.6774'}\n",
+      "Quantizing self_attn.k_proj in layer 11 of 24:  42%|████▏     | 10/24 [01:26<01:59,  8.54s/it]INFO - {'layer': 11, 'module': 'self_attn.k_proj', 'avg_loss': '0.0896', 'time': '0.2468'}\n",
+      "Quantizing self_attn.v_proj in layer 11 of 24:  42%|████▏     | 10/24 [01:27<01:59,  8.54s/it]INFO - {'layer': 11, 'module': 'self_attn.v_proj', 'avg_loss': '0.0336', 'time': '0.2502'}\n",
+      "Quantizing self_attn.q_proj in layer 11 of 24:  42%|████▏     | 10/24 [01:27<01:59,  8.54s/it]INFO - {'layer': 11, 'module': 'self_attn.q_proj', 'avg_loss': '0.0888', 'time': '0.2462'}\n",
+      "Quantizing self_attn.o_proj in layer 11 of 24:  42%|████▏     | 10/24 [01:28<01:59,  8.54s/it]INFO - {'layer': 11, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2457'}\n",
+      "Quantizing mlp.up_proj in layer 11 of 24:  42%|████▏     | 10/24 [01:29<01:59,  8.54s/it]     INFO - {'layer': 11, 'module': 'mlp.up_proj', 'avg_loss': '0.0564', 'time': '0.2464'}\n",
+      "Quantizing mlp.gate_proj in layer 11 of 24:  42%|████▏     | 10/24 [01:30<01:59,  8.54s/it]INFO - {'layer': 11, 'module': 'mlp.gate_proj', 'avg_loss': '0.0658', 'time': '0.2463'}\n",
+      "Quantizing mlp.down_proj in layer 11 of 24:  42%|████▏     | 10/24 [01:32<01:59,  8.54s/it]INFO - {'layer': 11, 'module': 'mlp.down_proj', 'avg_loss': '0.0022', 'time': '0.6804'}\n",
+      "Quantizing self_attn.k_proj in layer 12 of 24:  46%|████▌     | 11/24 [01:35<01:50,  8.53s/it]INFO - {'layer': 12, 'module': 'self_attn.k_proj', 'avg_loss': '0.0880', 'time': '0.2495'}\n",
+      "Quantizing self_attn.v_proj in layer 12 of 24:  46%|████▌     | 11/24 [01:35<01:50,  8.53s/it]INFO - {'layer': 12, 'module': 'self_attn.v_proj', 'avg_loss': '0.0369', 'time': '0.2485'}\n",
+      "Quantizing self_attn.q_proj in layer 12 of 24:  46%|████▌     | 11/24 [01:35<01:50,  8.53s/it]INFO - {'layer': 12, 'module': 'self_attn.q_proj', 'avg_loss': '0.0913', 'time': '0.2481'}\n",
+      "Quantizing self_attn.o_proj in layer 12 of 24:  46%|████▌     | 11/24 [01:37<01:50,  8.53s/it]INFO - {'layer': 12, 'module': 'self_attn.o_proj', 'avg_loss': '0.0022', 'time': '0.2462'}\n",
+      "Quantizing mlp.up_proj in layer 12 of 24:  46%|████▌     | 11/24 [01:38<01:50,  8.53s/it]     INFO - {'layer': 12, 'module': 'mlp.up_proj', 'avg_loss': '0.0626', 'time': '0.2482'}\n",
+      "Quantizing mlp.gate_proj in layer 12 of 24:  46%|████▌     | 11/24 [01:38<01:50,  8.53s/it]INFO - {'layer': 12, 'module': 'mlp.gate_proj', 'avg_loss': '0.0757', 'time': '0.2524'}\n",
+      "Quantizing mlp.down_proj in layer 12 of 24:  46%|████▌     | 11/24 [01:40<01:50,  8.53s/it]INFO - {'layer': 12, 'module': 'mlp.down_proj', 'avg_loss': '0.0027', 'time': '0.6790'}\n",
+      "Quantizing self_attn.k_proj in layer 13 of 24:  50%|█████     | 12/24 [01:43<01:42,  8.55s/it]INFO - {'layer': 13, 'module': 'self_attn.k_proj', 'avg_loss': '0.0972', 'time': '0.2483'}\n",
+      "Quantizing self_attn.v_proj in layer 13 of 24:  50%|█████     | 12/24 [01:44<01:42,  8.55s/it]INFO - {'layer': 13, 'module': 'self_attn.v_proj', 'avg_loss': '0.0364', 'time': '0.2508'}\n",
+      "Quantizing self_attn.q_proj in layer 13 of 24:  50%|█████     | 12/24 [01:44<01:42,  8.55s/it]INFO - {'layer': 13, 'module': 'self_attn.q_proj', 'avg_loss': '0.0959', 'time': '0.2523'}\n",
+      "Quantizing self_attn.o_proj in layer 13 of 24:  50%|█████     | 12/24 [01:45<01:42,  8.55s/it]INFO - {'layer': 13, 'module': 'self_attn.o_proj', 'avg_loss': '0.0030', 'time': '0.2469'}\n",
+      "Quantizing mlp.up_proj in layer 13 of 24:  50%|█████     | 12/24 [01:47<01:42,  8.55s/it]     INFO - {'layer': 13, 'module': 'mlp.up_proj', 'avg_loss': '0.0606', 'time': '0.2467'}\n",
+      "Quantizing mlp.gate_proj in layer 13 of 24:  50%|█████     | 12/24 [01:47<01:42,  8.55s/it]INFO - {'layer': 13, 'module': 'mlp.gate_proj', 'avg_loss': '0.0698', 'time': '0.2461'}\n",
+      "Quantizing mlp.down_proj in layer 13 of 24:  50%|█████     | 12/24 [01:49<01:42,  8.55s/it]INFO - {'layer': 13, 'module': 'mlp.down_proj', 'avg_loss': '0.0028', 'time': '0.6803'}\n",
+      "Quantizing self_attn.k_proj in layer 14 of 24:  54%|█████▍    | 13/24 [01:52<01:34,  8.58s/it]INFO - {'layer': 14, 'module': 'self_attn.k_proj', 'avg_loss': '0.0990', 'time': '0.2467'}\n",
+      "Quantizing self_attn.v_proj in layer 14 of 24:  54%|█████▍    | 13/24 [01:52<01:34,  8.58s/it]INFO - {'layer': 14, 'module': 'self_attn.v_proj', 'avg_loss': '0.0316', 'time': '0.2539'}\n",
+      "Quantizing self_attn.q_proj in layer 14 of 24:  54%|█████▍    | 13/24 [01:53<01:34,  8.58s/it]INFO - {'layer': 14, 'module': 'self_attn.q_proj', 'avg_loss': '0.0879', 'time': '0.2527'}\n",
+      "Quantizing self_attn.o_proj in layer 14 of 24:  54%|█████▍    | 13/24 [01:54<01:34,  8.58s/it]INFO - {'layer': 14, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2495'}\n",
+      "Quantizing mlp.up_proj in layer 14 of 24:  54%|█████▍    | 13/24 [01:55<01:34,  8.58s/it]     INFO - {'layer': 14, 'module': 'mlp.up_proj', 'avg_loss': '0.0633', 'time': '0.2473'}\n",
+      "Quantizing mlp.gate_proj in layer 14 of 24:  54%|█████▍    | 13/24 [01:55<01:34,  8.58s/it]INFO - {'layer': 14, 'module': 'mlp.gate_proj', 'avg_loss': '0.0739', 'time': '0.2488'}\n",
+      "Quantizing mlp.down_proj in layer 14 of 24:  54%|█████▍    | 13/24 [01:57<01:34,  8.58s/it]INFO - {'layer': 14, 'module': 'mlp.down_proj', 'avg_loss': '0.0028', 'time': '0.6782'}\n",
+      "Quantizing self_attn.k_proj in layer 15 of 24:  58%|█████▊    | 14/24 [02:01<01:26,  8.62s/it]INFO - {'layer': 15, 'module': 'self_attn.k_proj', 'avg_loss': '0.0921', 'time': '0.2470'}\n",
+      "Quantizing self_attn.v_proj in layer 15 of 24:  58%|█████▊    | 14/24 [02:01<01:26,  8.62s/it]INFO - {'layer': 15, 'module': 'self_attn.v_proj', 'avg_loss': '0.0395', 'time': '0.2504'}\n",
+      "Quantizing self_attn.q_proj in layer 15 of 24:  58%|█████▊    | 14/24 [02:01<01:26,  8.62s/it]INFO - {'layer': 15, 'module': 'self_attn.q_proj', 'avg_loss': '0.0865', 'time': '0.2485'}\n",
+      "Quantizing self_attn.o_proj in layer 15 of 24:  58%|█████▊    | 14/24 [02:02<01:26,  8.62s/it]INFO - {'layer': 15, 'module': 'self_attn.o_proj', 'avg_loss': '0.0026', 'time': '0.2451'}\n",
+      "Quantizing mlp.up_proj in layer 15 of 24:  58%|█████▊    | 14/24 [02:04<01:26,  8.62s/it]     INFO - {'layer': 15, 'module': 'mlp.up_proj', 'avg_loss': '0.0699', 'time': '0.2465'}\n",
+      "Quantizing mlp.gate_proj in layer 15 of 24:  58%|█████▊    | 14/24 [02:04<01:26,  8.62s/it]INFO - {'layer': 15, 'module': 'mlp.gate_proj', 'avg_loss': '0.0791', 'time': '0.2461'}\n",
+      "Quantizing mlp.down_proj in layer 15 of 24:  58%|█████▊    | 14/24 [02:06<01:26,  8.62s/it]INFO - {'layer': 15, 'module': 'mlp.down_proj', 'avg_loss': '0.0039', 'time': '0.6790'}\n",
+      "Quantizing self_attn.k_proj in layer 16 of 24:  62%|██████▎   | 15/24 [02:09<01:17,  8.57s/it]INFO - {'layer': 16, 'module': 'self_attn.k_proj', 'avg_loss': '0.0927', 'time': '0.2475'}\n",
+      "Quantizing self_attn.v_proj in layer 16 of 24:  62%|██████▎   | 15/24 [02:09<01:17,  8.57s/it]INFO - {'layer': 16, 'module': 'self_attn.v_proj', 'avg_loss': '0.0487', 'time': '0.2516'}\n",
+      "Quantizing self_attn.q_proj in layer 16 of 24:  62%|██████▎   | 15/24 [02:10<01:17,  8.57s/it]INFO - {'layer': 16, 'module': 'self_attn.q_proj', 'avg_loss': '0.0882', 'time': '0.2471'}\n",
+      "Quantizing self_attn.o_proj in layer 16 of 24:  62%|██████▎   | 15/24 [02:11<01:17,  8.57s/it]INFO - {'layer': 16, 'module': 'self_attn.o_proj', 'avg_loss': '0.0021', 'time': '0.2452'}\n",
+      "Quantizing mlp.up_proj in layer 16 of 24:  62%|██████▎   | 15/24 [02:12<01:17,  8.57s/it]     INFO - {'layer': 16, 'module': 'mlp.up_proj', 'avg_loss': '0.0842', 'time': '0.2463'}\n",
+      "Quantizing mlp.gate_proj in layer 16 of 24:  62%|██████▎   | 15/24 [02:13<01:17,  8.57s/it]INFO - {'layer': 16, 'module': 'mlp.gate_proj', 'avg_loss': '0.0926', 'time': '0.2461'}\n",
+      "Quantizing mlp.down_proj in layer 16 of 24:  62%|██████▎   | 15/24 [02:14<01:17,  8.57s/it]INFO - {'layer': 16, 'module': 'mlp.down_proj', 'avg_loss': '0.0057', 'time': '0.6790'}\n",
+      "Quantizing self_attn.k_proj in layer 17 of 24:  67%|██████▋   | 16/24 [02:18<01:08,  8.56s/it]INFO - {'layer': 17, 'module': 'self_attn.k_proj', 'avg_loss': '0.1059', 'time': '0.2480'}\n",
+      "Quantizing self_attn.v_proj in layer 17 of 24:  67%|██████▋   | 16/24 [02:18<01:08,  8.56s/it]INFO - {'layer': 17, 'module': 'self_attn.v_proj', 'avg_loss': '0.0595', 'time': '0.2482'}\n",
+      "Quantizing self_attn.q_proj in layer 17 of 24:  67%|██████▋   | 16/24 [02:18<01:08,  8.56s/it]INFO - {'layer': 17, 'module': 'self_attn.q_proj', 'avg_loss': '0.0968', 'time': '0.2466'}\n",
+      "Quantizing self_attn.o_proj in layer 17 of 24:  67%|██████▋   | 16/24 [02:19<01:08,  8.56s/it]INFO - {'layer': 17, 'module': 'self_attn.o_proj', 'avg_loss': '0.0027', 'time': '0.2455'}\n",
+      "Quantizing mlp.up_proj in layer 17 of 24:  67%|██████▋   | 16/24 [02:21<01:08,  8.56s/it]     INFO - {'layer': 17, 'module': 'mlp.up_proj', 'avg_loss': '0.1018', 'time': '0.2471'}\n",
+      "Quantizing mlp.gate_proj in layer 17 of 24:  67%|██████▋   | 16/24 [02:21<01:08,  8.56s/it]INFO - {'layer': 17, 'module': 'mlp.gate_proj', 'avg_loss': '0.1146', 'time': '0.2473'}\n",
+      "Quantizing mlp.down_proj in layer 17 of 24:  67%|██████▋   | 16/24 [02:23<01:08,  8.56s/it]INFO - {'layer': 17, 'module': 'mlp.down_proj', 'avg_loss': '0.0079', 'time': '0.6810'}\n",
+      "Quantizing self_attn.k_proj in layer 18 of 24:  71%|███████   | 17/24 [02:26<00:59,  8.51s/it]INFO - {'layer': 18, 'module': 'self_attn.k_proj', 'avg_loss': '0.1082', 'time': '0.2472'}\n",
+      "Quantizing self_attn.v_proj in layer 18 of 24:  71%|███████   | 17/24 [02:26<00:59,  8.51s/it]INFO - {'layer': 18, 'module': 'self_attn.v_proj', 'avg_loss': '0.0820', 'time': '0.2487'}\n",
+      "Quantizing self_attn.q_proj in layer 18 of 24:  71%|███████   | 17/24 [02:27<00:59,  8.51s/it]INFO - {'layer': 18, 'module': 'self_attn.q_proj', 'avg_loss': '0.1078', 'time': '0.2474'}\n",
+      "Quantizing self_attn.o_proj in layer 18 of 24:  71%|███████   | 17/24 [02:28<00:59,  8.51s/it]INFO - {'layer': 18, 'module': 'self_attn.o_proj', 'avg_loss': '0.0056', 'time': '0.2468'}\n",
+      "Quantizing mlp.up_proj in layer 18 of 24:  71%|███████   | 17/24 [02:29<00:59,  8.51s/it]     INFO - {'layer': 18, 'module': 'mlp.up_proj', 'avg_loss': '0.1236', 'time': '0.2455'}\n",
+      "Quantizing mlp.gate_proj in layer 18 of 24:  71%|███████   | 17/24 [02:30<00:59,  8.51s/it]INFO - {'layer': 18, 'module': 'mlp.gate_proj', 'avg_loss': '0.1284', 'time': '0.2460'}\n",
+      "Quantizing mlp.down_proj in layer 18 of 24:  71%|███████   | 17/24 [02:31<00:59,  8.51s/it]INFO - {'layer': 18, 'module': 'mlp.down_proj', 'avg_loss': '0.0112', 'time': '0.6785'}\n",
+      "Quantizing self_attn.k_proj in layer 19 of 24:  75%|███████▌  | 18/24 [02:35<00:51,  8.53s/it]INFO - {'layer': 19, 'module': 'self_attn.k_proj', 'avg_loss': '0.1101', 'time': '0.2465'}\n",
+      "Quantizing self_attn.v_proj in layer 19 of 24:  75%|███████▌  | 18/24 [02:35<00:51,  8.53s/it]INFO - {'layer': 19, 'module': 'self_attn.v_proj', 'avg_loss': '0.1003', 'time': '0.2485'}\n",
+      "Quantizing self_attn.q_proj in layer 19 of 24:  75%|███████▌  | 18/24 [02:35<00:51,  8.53s/it]INFO - {'layer': 19, 'module': 'self_attn.q_proj', 'avg_loss': '0.0979', 'time': '0.2466'}\n",
+      "Quantizing self_attn.o_proj in layer 19 of 24:  75%|███████▌  | 18/24 [02:36<00:51,  8.53s/it]INFO - {'layer': 19, 'module': 'self_attn.o_proj', 'avg_loss': '0.0040', 'time': '0.2453'}\n",
+      "Quantizing mlp.up_proj in layer 19 of 24:  75%|███████▌  | 18/24 [02:38<00:51,  8.53s/it]     INFO - {'layer': 19, 'module': 'mlp.up_proj', 'avg_loss': '0.1472', 'time': '0.2464'}\n",
+      "Quantizing mlp.gate_proj in layer 19 of 24:  75%|███████▌  | 18/24 [02:38<00:51,  8.53s/it]INFO - {'layer': 19, 'module': 'mlp.gate_proj', 'avg_loss': '0.1505', 'time': '0.2499'}\n",
+      "Quantizing mlp.down_proj in layer 19 of 24:  75%|███████▌  | 18/24 [02:40<00:51,  8.53s/it]INFO - {'layer': 19, 'module': 'mlp.down_proj', 'avg_loss': '0.0156', 'time': '0.6794'}\n",
+      "Quantizing self_attn.k_proj in layer 20 of 24:  79%|███████▉  | 19/24 [02:43<00:42,  8.52s/it]INFO - {'layer': 20, 'module': 'self_attn.k_proj', 'avg_loss': '0.1124', 'time': '0.2468'}\n",
+      "Quantizing self_attn.v_proj in layer 20 of 24:  79%|███████▉  | 19/24 [02:43<00:42,  8.52s/it]INFO - {'layer': 20, 'module': 'self_attn.v_proj', 'avg_loss': '0.1115', 'time': '0.2459'}\n",
+      "Quantizing self_attn.q_proj in layer 20 of 24:  79%|███████▉  | 19/24 [02:44<00:42,  8.52s/it]INFO - {'layer': 20, 'module': 'self_attn.q_proj', 'avg_loss': '0.1015', 'time': '0.2484'}\n",
+      "Quantizing self_attn.o_proj in layer 20 of 24:  79%|███████▉  | 19/24 [02:45<00:42,  8.52s/it]INFO - {'layer': 20, 'module': 'self_attn.o_proj', 'avg_loss': '0.0070', 'time': '0.2453'}\n",
+      "Quantizing mlp.up_proj in layer 20 of 24:  79%|███████▉  | 19/24 [02:46<00:42,  8.52s/it]     INFO - {'layer': 20, 'module': 'mlp.up_proj', 'avg_loss': '0.1550', 'time': '0.2450'}\n",
+      "Quantizing mlp.gate_proj in layer 20 of 24:  79%|███████▉  | 19/24 [02:47<00:42,  8.52s/it]INFO - {'layer': 20, 'module': 'mlp.gate_proj', 'avg_loss': '0.1512', 'time': '0.2451'}\n",
+      "Quantizing mlp.down_proj in layer 20 of 24:  79%|███████▉  | 19/24 [02:49<00:42,  8.52s/it]INFO - {'layer': 20, 'module': 'mlp.down_proj', 'avg_loss': '0.0180', 'time': '0.6781'}\n",
+      "Quantizing self_attn.k_proj in layer 21 of 24:  83%|████████▎ | 20/24 [02:52<00:34,  8.57s/it]INFO - {'layer': 21, 'module': 'self_attn.k_proj', 'avg_loss': '0.1197', 'time': '0.2465'}\n",
+      "Quantizing self_attn.v_proj in layer 21 of 24:  83%|████████▎ | 20/24 [02:52<00:34,  8.57s/it]INFO - {'layer': 21, 'module': 'self_attn.v_proj', 'avg_loss': '0.1370', 'time': '0.2484'}\n",
+      "Quantizing self_attn.q_proj in layer 21 of 24:  83%|████████▎ | 20/24 [02:52<00:34,  8.57s/it]INFO - {'layer': 21, 'module': 'self_attn.q_proj', 'avg_loss': '0.1034', 'time': '0.2461'}\n",
+      "Quantizing self_attn.o_proj in layer 21 of 24:  83%|████████▎ | 20/24 [02:54<00:34,  8.57s/it]INFO - {'layer': 21, 'module': 'self_attn.o_proj', 'avg_loss': '0.0058', 'time': '0.2492'}\n",
+      "Quantizing mlp.up_proj in layer 21 of 24:  83%|████████▎ | 20/24 [02:55<00:34,  8.57s/it]     INFO - {'layer': 21, 'module': 'mlp.up_proj', 'avg_loss': '0.1678', 'time': '0.2464'}\n",
+      "Quantizing mlp.gate_proj in layer 21 of 24:  83%|████████▎ | 20/24 [02:55<00:34,  8.57s/it]INFO - {'layer': 21, 'module': 'mlp.gate_proj', 'avg_loss': '0.1542', 'time': '0.2494'}\n",
+      "Quantizing mlp.down_proj in layer 21 of 24:  83%|████████▎ | 20/24 [02:57<00:34,  8.57s/it]INFO - {'layer': 21, 'module': 'mlp.down_proj', 'avg_loss': '0.0229', 'time': '0.6776'}\n",
+      "Quantizing self_attn.k_proj in layer 22 of 24:  88%|████████▊ | 21/24 [03:00<00:25,  8.56s/it]INFO - {'layer': 22, 'module': 'self_attn.k_proj', 'avg_loss': '0.1123', 'time': '0.2468'}\n",
+      "Quantizing self_attn.v_proj in layer 22 of 24:  88%|████████▊ | 21/24 [03:01<00:25,  8.56s/it]INFO - {'layer': 22, 'module': 'self_attn.v_proj', 'avg_loss': '0.1440', 'time': '0.2480'}\n",
+      "Quantizing self_attn.q_proj in layer 22 of 24:  88%|████████▊ | 21/24 [03:01<00:25,  8.56s/it]INFO - {'layer': 22, 'module': 'self_attn.q_proj', 'avg_loss': '0.1030', 'time': '0.2464'}\n",
+      "Quantizing self_attn.o_proj in layer 22 of 24:  88%|████████▊ | 21/24 [03:02<00:25,  8.56s/it]INFO - {'layer': 22, 'module': 'self_attn.o_proj', 'avg_loss': '0.0116', 'time': '0.2450'}\n",
+      "Quantizing mlp.up_proj in layer 22 of 24:  88%|████████▊ | 21/24 [03:04<00:25,  8.56s/it]     INFO - {'layer': 22, 'module': 'mlp.up_proj', 'avg_loss': '0.1743', 'time': '0.2452'}\n",
+      "Quantizing mlp.gate_proj in layer 22 of 24:  88%|████████▊ | 21/24 [03:04<00:25,  8.56s/it]INFO - {'layer': 22, 'module': 'mlp.gate_proj', 'avg_loss': '0.1545', 'time': '0.2481'}\n",
+      "Quantizing mlp.down_proj in layer 22 of 24:  88%|████████▊ | 21/24 [03:06<00:25,  8.56s/it]INFO - {'layer': 22, 'module': 'mlp.down_proj', 'avg_loss': '0.0335', 'time': '0.6765'}\n",
+      "Quantizing self_attn.k_proj in layer 23 of 24:  92%|█████████▏| 22/24 [03:09<00:17,  8.57s/it]INFO - {'layer': 23, 'module': 'self_attn.k_proj', 'avg_loss': '0.1069', 'time': '0.2518'}\n",
+      "Quantizing self_attn.v_proj in layer 23 of 24:  92%|█████████▏| 22/24 [03:09<00:17,  8.57s/it]INFO - {'layer': 23, 'module': 'self_attn.v_proj', 'avg_loss': '0.1511', 'time': '0.2485'}\n",
+      "Quantizing self_attn.q_proj in layer 23 of 24:  92%|█████████▏| 22/24 [03:10<00:17,  8.57s/it]INFO - {'layer': 23, 'module': 'self_attn.q_proj', 'avg_loss': '0.1324', 'time': '0.2463'}\n",
+      "Quantizing self_attn.o_proj in layer 23 of 24:  92%|█████████▏| 22/24 [03:11<00:17,  8.57s/it]INFO - {'layer': 23, 'module': 'self_attn.o_proj', 'avg_loss': '0.0338', 'time': '0.2454'}\n",
+      "Quantizing mlp.up_proj in layer 23 of 24:  92%|█████████▏| 22/24 [03:12<00:17,  8.57s/it]     INFO - {'layer': 23, 'module': 'mlp.up_proj', 'avg_loss': '0.1691', 'time': '0.2470'}\n",
+      "Quantizing mlp.gate_proj in layer 23 of 24:  92%|█████████▏| 22/24 [03:12<00:17,  8.57s/it]INFO - {'layer': 23, 'module': 'mlp.gate_proj', 'avg_loss': '0.1514', 'time': '0.2481'}\n",
+      "Quantizing mlp.down_proj in layer 23 of 24:  92%|█████████▏| 22/24 [03:14<00:17,  8.57s/it]INFO - {'layer': 23, 'module': 'mlp.down_proj', 'avg_loss': '0.1119', 'time': '0.6800'}\n",
+      "Quantizing self_attn.k_proj in layer 24 of 24:  96%|█████████▌| 23/24 [03:17<00:08,  8.52s/it]INFO - {'layer': 24, 'module': 'self_attn.k_proj', 'avg_loss': '0.0888', 'time': '0.2469'}\n",
+      "Quantizing self_attn.v_proj in layer 24 of 24:  96%|█████████▌| 23/24 [03:18<00:08,  8.52s/it]INFO - {'layer': 24, 'module': 'self_attn.v_proj', 'avg_loss': '0.1012', 'time': '0.2473'}\n",
+      "Quantizing self_attn.q_proj in layer 24 of 24:  96%|█████████▌| 23/24 [03:18<00:08,  8.52s/it]INFO - {'layer': 24, 'module': 'self_attn.q_proj', 'avg_loss': '0.0848', 'time': '0.2462'}\n",
+      "Quantizing self_attn.o_proj in layer 24 of 24:  96%|█████████▌| 23/24 [03:19<00:08,  8.52s/it]INFO - {'layer': 24, 'module': 'self_attn.o_proj', 'avg_loss': '0.0100', 'time': '0.2472'}\n",
+      "Quantizing mlp.up_proj in layer 24 of 24:  96%|█████████▌| 23/24 [03:21<00:08,  8.52s/it]     INFO - {'layer': 24, 'module': 'mlp.up_proj', 'avg_loss': '0.2124', 'time': '0.2458'}\n",
+      "Quantizing mlp.gate_proj in layer 24 of 24:  96%|█████████▌| 23/24 [03:21<00:08,  8.52s/it]INFO - {'layer': 24, 'module': 'mlp.gate_proj', 'avg_loss': '0.1940', 'time': '0.2455'}\n",
+      "Quantizing mlp.down_proj in layer 24 of 24:  96%|█████████▌| 23/24 [03:23<00:08,  8.52s/it]INFO - {'layer': 24, 'module': 'mlp.down_proj', 'avg_loss': '0.1381', 'time': '0.6771'}\n",
+      "Quantizing mlp.down_proj in layer 24 of 24: 100%|██████████| 24/24 [03:25<00:00,  8.55s/it]\n",
+      "INFO - Quantization summary:\n",
+      "[{'layer': 1, 'module': 'self_attn.k_proj', 'avg_loss': '0.0039', 'time': '0.2490'}, {'layer': 1, 'module': 'self_attn.v_proj', 'avg_loss': '0.0001', 'time': '0.2508'}, {'layer': 1, 'module': 'self_attn.q_proj', 'avg_loss': '0.0058', 'time': '0.2535'}, {'layer': 1, 'module': 'self_attn.o_proj', 'avg_loss': '0.0000', 'time': '0.2501'}, {'layer': 1, 'module': 'mlp.up_proj', 'avg_loss': '0.0183', 'time': '0.2542'}, {'layer': 1, 'module': 'mlp.gate_proj', 'avg_loss': '0.0249', 'time': '0.2524'}, {'layer': 1, 'module': 'mlp.down_proj', 'avg_loss': '0.0009', 'time': '0.7110'}, {'layer': 2, 'module': 'self_attn.k_proj', 'avg_loss': '0.0487', 'time': '0.2542'}, {'layer': 2, 'module': 'self_attn.v_proj', 'avg_loss': '0.0056', 'time': '0.2556'}, {'layer': 2, 'module': 'self_attn.q_proj', 'avg_loss': '0.0434', 'time': '0.2489'}, {'layer': 2, 'module': 'self_attn.o_proj', 'avg_loss': '0.0001', 'time': '0.2569'}, {'layer': 2, 'module': 'mlp.up_proj', 'avg_loss': '0.0265', 'time': '0.2470'}, {'layer': 2, 'module': 'mlp.gate_proj', 'avg_loss': '0.0431', 'time': '0.2478'}, {'layer': 2, 'module': 'mlp.down_proj', 'avg_loss': '0.0006', 'time': '0.6856'}, {'layer': 3, 'module': 'self_attn.k_proj', 'avg_loss': '0.0950', 'time': '0.2479'}, {'layer': 3, 'module': 'self_attn.v_proj', 'avg_loss': '0.0140', 'time': '0.2482'}, {'layer': 3, 'module': 'self_attn.q_proj', 'avg_loss': '0.0834', 'time': '0.2478'}, {'layer': 3, 'module': 'self_attn.o_proj', 'avg_loss': '0.0002', 'time': '0.2489'}, {'layer': 3, 'module': 'mlp.up_proj', 'avg_loss': '0.0338', 'time': '0.2457'}, {'layer': 3, 'module': 'mlp.gate_proj', 'avg_loss': '0.0550', 'time': '0.2464'}, {'layer': 3, 'module': 'mlp.down_proj', 'avg_loss': '0.0009', 'time': '0.6789'}, {'layer': 4, 'module': 'self_attn.k_proj', 'avg_loss': '0.0903', 'time': '0.2479'}, {'layer': 4, 'module': 'self_attn.v_proj', 'avg_loss': '0.0177', 'time': '0.2490'}, {'layer': 4, 'module': 'self_attn.q_proj', 'avg_loss': '0.0796', 'time': '0.2491'}, {'layer': 4, 'module': 'self_attn.o_proj', 'avg_loss': '0.0003', 'time': '0.2473'}, {'layer': 4, 'module': 'mlp.up_proj', 'avg_loss': '0.0439', 'time': '0.2482'}, {'layer': 4, 'module': 'mlp.gate_proj', 'avg_loss': '0.0609', 'time': '0.2463'}, {'layer': 4, 'module': 'mlp.down_proj', 'avg_loss': '0.0013', 'time': '0.6797'}, {'layer': 5, 'module': 'self_attn.k_proj', 'avg_loss': '0.0705', 'time': '0.2469'}, {'layer': 5, 'module': 'self_attn.v_proj', 'avg_loss': '0.0124', 'time': '0.2492'}, {'layer': 5, 'module': 'self_attn.q_proj', 'avg_loss': '0.0630', 'time': '0.2472'}, {'layer': 5, 'module': 'self_attn.o_proj', 'avg_loss': '0.0004', 'time': '0.2458'}, {'layer': 5, 'module': 'mlp.up_proj', 'avg_loss': '0.0506', 'time': '0.2469'}, {'layer': 5, 'module': 'mlp.gate_proj', 'avg_loss': '0.0733', 'time': '0.2465'}, {'layer': 5, 'module': 'mlp.down_proj', 'avg_loss': '0.0014', 'time': '0.6803'}, {'layer': 6, 'module': 'self_attn.k_proj', 'avg_loss': '0.0940', 'time': '0.2497'}, {'layer': 6, 'module': 'self_attn.v_proj', 'avg_loss': '0.0180', 'time': '0.2521'}, {'layer': 6, 'module': 'self_attn.q_proj', 'avg_loss': '0.0856', 'time': '0.2488'}, {'layer': 6, 'module': 'self_attn.o_proj', 'avg_loss': '0.0007', 'time': '0.2459'}, {'layer': 6, 'module': 'mlp.up_proj', 'avg_loss': '0.0614', 'time': '0.2470'}, {'layer': 6, 'module': 'mlp.gate_proj', 'avg_loss': '0.0813', 'time': '0.2456'}, {'layer': 6, 'module': 'mlp.down_proj', 'avg_loss': '0.0075', 'time': '0.6766'}, {'layer': 7, 'module': 'self_attn.k_proj', 'avg_loss': '0.0902', 'time': '0.2492'}, {'layer': 7, 'module': 'self_attn.v_proj', 'avg_loss': '0.0270', 'time': '0.2521'}, {'layer': 7, 'module': 'self_attn.q_proj', 'avg_loss': '0.0869', 'time': '0.2503'}, {'layer': 7, 'module': 'self_attn.o_proj', 'avg_loss': '0.0008', 'time': '0.2488'}, {'layer': 7, 'module': 'mlp.up_proj', 'avg_loss': '0.0667', 'time': '0.2496'}, {'layer': 7, 'module': 'mlp.gate_proj', 'avg_loss': '0.0965', 'time': '0.2494'}, {'layer': 7, 'module': 'mlp.down_proj', 'avg_loss': '0.0022', 'time': '0.6853'}, {'layer': 8, 'module': 'self_attn.k_proj', 'avg_loss': '0.0968', 'time': '0.2501'}, {'layer': 8, 'module': 'self_attn.v_proj', 'avg_loss': '0.0316', 'time': '0.2510'}, {'layer': 8, 'module': 'self_attn.q_proj', 'avg_loss': '0.0997', 'time': '0.2578'}, {'layer': 8, 'module': 'self_attn.o_proj', 'avg_loss': '0.0010', 'time': '0.2455'}, {'layer': 8, 'module': 'mlp.up_proj', 'avg_loss': '0.0680', 'time': '0.2488'}, {'layer': 8, 'module': 'mlp.gate_proj', 'avg_loss': '0.0904', 'time': '0.2472'}, {'layer': 8, 'module': 'mlp.down_proj', 'avg_loss': '0.0024', 'time': '0.6778'}, {'layer': 9, 'module': 'self_attn.k_proj', 'avg_loss': '0.0799', 'time': '0.2464'}, {'layer': 9, 'module': 'self_attn.v_proj', 'avg_loss': '0.0242', 'time': '0.2489'}, {'layer': 9, 'module': 'self_attn.q_proj', 'avg_loss': '0.0754', 'time': '0.2512'}, {'layer': 9, 'module': 'self_attn.o_proj', 'avg_loss': '0.0012', 'time': '0.2452'}, {'layer': 9, 'module': 'mlp.up_proj', 'avg_loss': '0.0668', 'time': '0.2462'}, {'layer': 9, 'module': 'mlp.gate_proj', 'avg_loss': '0.0844', 'time': '0.2466'}, {'layer': 9, 'module': 'mlp.down_proj', 'avg_loss': '0.0025', 'time': '0.6810'}, {'layer': 10, 'module': 'self_attn.k_proj', 'avg_loss': '0.0775', 'time': '0.2490'}, {'layer': 10, 'module': 'self_attn.v_proj', 'avg_loss': '0.0229', 'time': '0.2503'}, {'layer': 10, 'module': 'self_attn.q_proj', 'avg_loss': '0.0778', 'time': '0.2487'}, {'layer': 10, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2569'}, {'layer': 10, 'module': 'mlp.up_proj', 'avg_loss': '0.0604', 'time': '0.2461'}, {'layer': 10, 'module': 'mlp.gate_proj', 'avg_loss': '0.0760', 'time': '0.2470'}, {'layer': 10, 'module': 'mlp.down_proj', 'avg_loss': '0.0021', 'time': '0.6774'}, {'layer': 11, 'module': 'self_attn.k_proj', 'avg_loss': '0.0896', 'time': '0.2468'}, {'layer': 11, 'module': 'self_attn.v_proj', 'avg_loss': '0.0336', 'time': '0.2502'}, {'layer': 11, 'module': 'self_attn.q_proj', 'avg_loss': '0.0888', 'time': '0.2462'}, {'layer': 11, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2457'}, {'layer': 11, 'module': 'mlp.up_proj', 'avg_loss': '0.0564', 'time': '0.2464'}, {'layer': 11, 'module': 'mlp.gate_proj', 'avg_loss': '0.0658', 'time': '0.2463'}, {'layer': 11, 'module': 'mlp.down_proj', 'avg_loss': '0.0022', 'time': '0.6804'}, {'layer': 12, 'module': 'self_attn.k_proj', 'avg_loss': '0.0880', 'time': '0.2495'}, {'layer': 12, 'module': 'self_attn.v_proj', 'avg_loss': '0.0369', 'time': '0.2485'}, {'layer': 12, 'module': 'self_attn.q_proj', 'avg_loss': '0.0913', 'time': '0.2481'}, {'layer': 12, 'module': 'self_attn.o_proj', 'avg_loss': '0.0022', 'time': '0.2462'}, {'layer': 12, 'module': 'mlp.up_proj', 'avg_loss': '0.0626', 'time': '0.2482'}, {'layer': 12, 'module': 'mlp.gate_proj', 'avg_loss': '0.0757', 'time': '0.2524'}, {'layer': 12, 'module': 'mlp.down_proj', 'avg_loss': '0.0027', 'time': '0.6790'}, {'layer': 13, 'module': 'self_attn.k_proj', 'avg_loss': '0.0972', 'time': '0.2483'}, {'layer': 13, 'module': 'self_attn.v_proj', 'avg_loss': '0.0364', 'time': '0.2508'}, {'layer': 13, 'module': 'self_attn.q_proj', 'avg_loss': '0.0959', 'time': '0.2523'}, {'layer': 13, 'module': 'self_attn.o_proj', 'avg_loss': '0.0030', 'time': '0.2469'}, {'layer': 13, 'module': 'mlp.up_proj', 'avg_loss': '0.0606', 'time': '0.2467'}, {'layer': 13, 'module': 'mlp.gate_proj', 'avg_loss': '0.0698', 'time': '0.2461'}, {'layer': 13, 'module': 'mlp.down_proj', 'avg_loss': '0.0028', 'time': '0.6803'}, {'layer': 14, 'module': 'self_attn.k_proj', 'avg_loss': '0.0990', 'time': '0.2467'}, {'layer': 14, 'module': 'self_attn.v_proj', 'avg_loss': '0.0316', 'time': '0.2539'}, {'layer': 14, 'module': 'self_attn.q_proj', 'avg_loss': '0.0879', 'time': '0.2527'}, {'layer': 14, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2495'}, {'layer': 14, 'module': 'mlp.up_proj', 'avg_loss': '0.0633', 'time': '0.2473'}, {'layer': 14, 'module': 'mlp.gate_proj', 'avg_loss': '0.0739', 'time': '0.2488'}, {'layer': 14, 'module': 'mlp.down_proj', 'avg_loss': '0.0028', 'time': '0.6782'}, {'layer': 15, 'module': 'self_attn.k_proj', 'avg_loss': '0.0921', 'time': '0.2470'}, {'layer': 15, 'module': 'self_attn.v_proj', 'avg_loss': '0.0395', 'time': '0.2504'}, {'layer': 15, 'module': 'self_attn.q_proj', 'avg_loss': '0.0865', 'time': '0.2485'}, {'layer': 15, 'module': 'self_attn.o_proj', 'avg_loss': '0.0026', 'time': '0.2451'}, {'layer': 15, 'module': 'mlp.up_proj', 'avg_loss': '0.0699', 'time': '0.2465'}, {'layer': 15, 'module': 'mlp.gate_proj', 'avg_loss': '0.0791', 'time': '0.2461'}, {'layer': 15, 'module': 'mlp.down_proj', 'avg_loss': '0.0039', 'time': '0.6790'}, {'layer': 16, 'module': 'self_attn.k_proj', 'avg_loss': '0.0927', 'time': '0.2475'}, {'layer': 16, 'module': 'self_attn.v_proj', 'avg_loss': '0.0487', 'time': '0.2516'}, {'layer': 16, 'module': 'self_attn.q_proj', 'avg_loss': '0.0882', 'time': '0.2471'}, {'layer': 16, 'module': 'self_attn.o_proj', 'avg_loss': '0.0021', 'time': '0.2452'}, {'layer': 16, 'module': 'mlp.up_proj', 'avg_loss': '0.0842', 'time': '0.2463'}, {'layer': 16, 'module': 'mlp.gate_proj', 'avg_loss': '0.0926', 'time': '0.2461'}, {'layer': 16, 'module': 'mlp.down_proj', 'avg_loss': '0.0057', 'time': '0.6790'}, {'layer': 17, 'module': 'self_attn.k_proj', 'avg_loss': '0.1059', 'time': '0.2480'}, {'layer': 17, 'module': 'self_attn.v_proj', 'avg_loss': '0.0595', 'time': '0.2482'}, {'layer': 17, 'module': 'self_attn.q_proj', 'avg_loss': '0.0968', 'time': '0.2466'}, {'layer': 17, 'module': 'self_attn.o_proj', 'avg_loss': '0.0027', 'time': '0.2455'}, {'layer': 17, 'module': 'mlp.up_proj', 'avg_loss': '0.1018', 'time': '0.2471'}, {'layer': 17, 'module': 'mlp.gate_proj', 'avg_loss': '0.1146', 'time': '0.2473'}, {'layer': 17, 'module': 'mlp.down_proj', 'avg_loss': '0.0079', 'time': '0.6810'}, {'layer': 18, 'module': 'self_attn.k_proj', 'avg_loss': '0.1082', 'time': '0.2472'}, {'layer': 18, 'module': 'self_attn.v_proj', 'avg_loss': '0.0820', 'time': '0.2487'}, {'layer': 18, 'module': 'self_attn.q_proj', 'avg_loss': '0.1078', 'time': '0.2474'}, {'layer': 18, 'module': 'self_attn.o_proj', 'avg_loss': '0.0056', 'time': '0.2468'}, {'layer': 18, 'module': 'mlp.up_proj', 'avg_loss': '0.1236', 'time': '0.2455'}, {'layer': 18, 'module': 'mlp.gate_proj', 'avg_loss': '0.1284', 'time': '0.2460'}, {'layer': 18, 'module': 'mlp.down_proj', 'avg_loss': '0.0112', 'time': '0.6785'}, {'layer': 19, 'module': 'self_attn.k_proj', 'avg_loss': '0.1101', 'time': '0.2465'}, {'layer': 19, 'module': 'self_attn.v_proj', 'avg_loss': '0.1003', 'time': '0.2485'}, {'layer': 19, 'module': 'self_attn.q_proj', 'avg_loss': '0.0979', 'time': '0.2466'}, {'layer': 19, 'module': 'self_attn.o_proj', 'avg_loss': '0.0040', 'time': '0.2453'}, {'layer': 19, 'module': 'mlp.up_proj', 'avg_loss': '0.1472', 'time': '0.2464'}, {'layer': 19, 'module': 'mlp.gate_proj', 'avg_loss': '0.1505', 'time': '0.2499'}, {'layer': 19, 'module': 'mlp.down_proj', 'avg_loss': '0.0156', 'time': '0.6794'}, {'layer': 20, 'module': 'self_attn.k_proj', 'avg_loss': '0.1124', 'time': '0.2468'}, {'layer': 20, 'module': 'self_attn.v_proj', 'avg_loss': '0.1115', 'time': '0.2459'}, {'layer': 20, 'module': 'self_attn.q_proj', 'avg_loss': '0.1015', 'time': '0.2484'}, {'layer': 20, 'module': 'self_attn.o_proj', 'avg_loss': '0.0070', 'time': '0.2453'}, {'layer': 20, 'module': 'mlp.up_proj', 'avg_loss': '0.1550', 'time': '0.2450'}, {'layer': 20, 'module': 'mlp.gate_proj', 'avg_loss': '0.1512', 'time': '0.2451'}, {'layer': 20, 'module': 'mlp.down_proj', 'avg_loss': '0.0180', 'time': '0.6781'}, {'layer': 21, 'module': 'self_attn.k_proj', 'avg_loss': '0.1197', 'time': '0.2465'}, {'layer': 21, 'module': 'self_attn.v_proj', 'avg_loss': '0.1370', 'time': '0.2484'}, {'layer': 21, 'module': 'self_attn.q_proj', 'avg_loss': '0.1034', 'time': '0.2461'}, {'layer': 21, 'module': 'self_attn.o_proj', 'avg_loss': '0.0058', 'time': '0.2492'}, {'layer': 21, 'module': 'mlp.up_proj', 'avg_loss': '0.1678', 'time': '0.2464'}, {'layer': 21, 'module': 'mlp.gate_proj', 'avg_loss': '0.1542', 'time': '0.2494'}, {'layer': 21, 'module': 'mlp.down_proj', 'avg_loss': '0.0229', 'time': '0.6776'}, {'layer': 22, 'module': 'self_attn.k_proj', 'avg_loss': '0.1123', 'time': '0.2468'}, {'layer': 22, 'module': 'self_attn.v_proj', 'avg_loss': '0.1440', 'time': '0.2480'}, {'layer': 22, 'module': 'self_attn.q_proj', 'avg_loss': '0.1030', 'time': '0.2464'}, {'layer': 22, 'module': 'self_attn.o_proj', 'avg_loss': '0.0116', 'time': '0.2450'}, {'layer': 22, 'module': 'mlp.up_proj', 'avg_loss': '0.1743', 'time': '0.2452'}, {'layer': 22, 'module': 'mlp.gate_proj', 'avg_loss': '0.1545', 'time': '0.2481'}, {'layer': 22, 'module': 'mlp.down_proj', 'avg_loss': '0.0335', 'time': '0.6765'}, {'layer': 23, 'module': 'self_attn.k_proj', 'avg_loss': '0.1069', 'time': '0.2518'}, {'layer': 23, 'module': 'self_attn.v_proj', 'avg_loss': '0.1511', 'time': '0.2485'}, {'layer': 23, 'module': 'self_attn.q_proj', 'avg_loss': '0.1324', 'time': '0.2463'}, {'layer': 23, 'module': 'self_attn.o_proj', 'avg_loss': '0.0338', 'time': '0.2454'}, {'layer': 23, 'module': 'mlp.up_proj', 'avg_loss': '0.1691', 'time': '0.2470'}, {'layer': 23, 'module': 'mlp.gate_proj', 'avg_loss': '0.1514', 'time': '0.2481'}, {'layer': 23, 'module': 'mlp.down_proj', 'avg_loss': '0.1119', 'time': '0.6800'}, {'layer': 24, 'module': 'self_attn.k_proj', 'avg_loss': '0.0888', 'time': '0.2469'}, {'layer': 24, 'module': 'self_attn.v_proj', 'avg_loss': '0.1012', 'time': '0.2473'}, {'layer': 24, 'module': 'self_attn.q_proj', 'avg_loss': '0.0848', 'time': '0.2462'}, {'layer': 24, 'module': 'self_attn.o_proj', 'avg_loss': '0.0100', 'time': '0.2472'}, {'layer': 24, 'module': 'mlp.up_proj', 'avg_loss': '0.2124', 'time': '0.2458'}, {'layer': 24, 'module': 'mlp.gate_proj', 'avg_loss': '0.1940', 'time': '0.2455'}, {'layer': 24, 'module': 'mlp.down_proj', 'avg_loss': '0.1381', 'time': '0.6771'}]\n",
+      "INFO - {'layer': 1, 'module': 'self_attn.k_proj', 'avg_loss': '0.0039', 'time': '0.2490'}\n",
+      "INFO - {'layer': 1, 'module': 'self_attn.v_proj', 'avg_loss': '0.0001', 'time': '0.2508'}\n",
+      "INFO - {'layer': 1, 'module': 'self_attn.q_proj', 'avg_loss': '0.0058', 'time': '0.2535'}\n",
+      "INFO - {'layer': 1, 'module': 'self_attn.o_proj', 'avg_loss': '0.0000', 'time': '0.2501'}\n",
+      "INFO - {'layer': 1, 'module': 'mlp.up_proj', 'avg_loss': '0.0183', 'time': '0.2542'}\n",
+      "INFO - {'layer': 1, 'module': 'mlp.gate_proj', 'avg_loss': '0.0249', 'time': '0.2524'}\n",
+      "INFO - {'layer': 1, 'module': 'mlp.down_proj', 'avg_loss': '0.0009', 'time': '0.7110'}\n",
+      "INFO - {'layer': 2, 'module': 'self_attn.k_proj', 'avg_loss': '0.0487', 'time': '0.2542'}\n",
+      "INFO - {'layer': 2, 'module': 'self_attn.v_proj', 'avg_loss': '0.0056', 'time': '0.2556'}\n",
+      "INFO - {'layer': 2, 'module': 'self_attn.q_proj', 'avg_loss': '0.0434', 'time': '0.2489'}\n",
+      "INFO - {'layer': 2, 'module': 'self_attn.o_proj', 'avg_loss': '0.0001', 'time': '0.2569'}\n",
+      "INFO - {'layer': 2, 'module': 'mlp.up_proj', 'avg_loss': '0.0265', 'time': '0.2470'}\n",
+      "INFO - {'layer': 2, 'module': 'mlp.gate_proj', 'avg_loss': '0.0431', 'time': '0.2478'}\n",
+      "INFO - {'layer': 2, 'module': 'mlp.down_proj', 'avg_loss': '0.0006', 'time': '0.6856'}\n",
+      "INFO - {'layer': 3, 'module': 'self_attn.k_proj', 'avg_loss': '0.0950', 'time': '0.2479'}\n",
+      "INFO - {'layer': 3, 'module': 'self_attn.v_proj', 'avg_loss': '0.0140', 'time': '0.2482'}\n",
+      "INFO - {'layer': 3, 'module': 'self_attn.q_proj', 'avg_loss': '0.0834', 'time': '0.2478'}\n",
+      "INFO - {'layer': 3, 'module': 'self_attn.o_proj', 'avg_loss': '0.0002', 'time': '0.2489'}\n",
+      "INFO - {'layer': 3, 'module': 'mlp.up_proj', 'avg_loss': '0.0338', 'time': '0.2457'}\n",
+      "INFO - {'layer': 3, 'module': 'mlp.gate_proj', 'avg_loss': '0.0550', 'time': '0.2464'}\n",
+      "INFO - {'layer': 3, 'module': 'mlp.down_proj', 'avg_loss': '0.0009', 'time': '0.6789'}\n",
+      "INFO - {'layer': 4, 'module': 'self_attn.k_proj', 'avg_loss': '0.0903', 'time': '0.2479'}\n",
+      "INFO - {'layer': 4, 'module': 'self_attn.v_proj', 'avg_loss': '0.0177', 'time': '0.2490'}\n",
+      "INFO - {'layer': 4, 'module': 'self_attn.q_proj', 'avg_loss': '0.0796', 'time': '0.2491'}\n",
+      "INFO - {'layer': 4, 'module': 'self_attn.o_proj', 'avg_loss': '0.0003', 'time': '0.2473'}\n",
+      "INFO - {'layer': 4, 'module': 'mlp.up_proj', 'avg_loss': '0.0439', 'time': '0.2482'}\n",
+      "INFO - {'layer': 4, 'module': 'mlp.gate_proj', 'avg_loss': '0.0609', 'time': '0.2463'}\n",
+      "INFO - {'layer': 4, 'module': 'mlp.down_proj', 'avg_loss': '0.0013', 'time': '0.6797'}\n",
+      "INFO - {'layer': 5, 'module': 'self_attn.k_proj', 'avg_loss': '0.0705', 'time': '0.2469'}\n",
+      "INFO - {'layer': 5, 'module': 'self_attn.v_proj', 'avg_loss': '0.0124', 'time': '0.2492'}\n",
+      "INFO - {'layer': 5, 'module': 'self_attn.q_proj', 'avg_loss': '0.0630', 'time': '0.2472'}\n",
+      "INFO - {'layer': 5, 'module': 'self_attn.o_proj', 'avg_loss': '0.0004', 'time': '0.2458'}\n",
+      "INFO - {'layer': 5, 'module': 'mlp.up_proj', 'avg_loss': '0.0506', 'time': '0.2469'}\n",
+      "INFO - {'layer': 5, 'module': 'mlp.gate_proj', 'avg_loss': '0.0733', 'time': '0.2465'}\n",
+      "INFO - {'layer': 5, 'module': 'mlp.down_proj', 'avg_loss': '0.0014', 'time': '0.6803'}\n",
+      "INFO - {'layer': 6, 'module': 'self_attn.k_proj', 'avg_loss': '0.0940', 'time': '0.2497'}\n",
+      "INFO - {'layer': 6, 'module': 'self_attn.v_proj', 'avg_loss': '0.0180', 'time': '0.2521'}\n",
+      "INFO - {'layer': 6, 'module': 'self_attn.q_proj', 'avg_loss': '0.0856', 'time': '0.2488'}\n",
+      "INFO - {'layer': 6, 'module': 'self_attn.o_proj', 'avg_loss': '0.0007', 'time': '0.2459'}\n",
+      "INFO - {'layer': 6, 'module': 'mlp.up_proj', 'avg_loss': '0.0614', 'time': '0.2470'}\n",
+      "INFO - {'layer': 6, 'module': 'mlp.gate_proj', 'avg_loss': '0.0813', 'time': '0.2456'}\n",
+      "INFO - {'layer': 6, 'module': 'mlp.down_proj', 'avg_loss': '0.0075', 'time': '0.6766'}\n",
+      "INFO - {'layer': 7, 'module': 'self_attn.k_proj', 'avg_loss': '0.0902', 'time': '0.2492'}\n",
+      "INFO - {'layer': 7, 'module': 'self_attn.v_proj', 'avg_loss': '0.0270', 'time': '0.2521'}\n",
+      "INFO - {'layer': 7, 'module': 'self_attn.q_proj', 'avg_loss': '0.0869', 'time': '0.2503'}\n",
+      "INFO - {'layer': 7, 'module': 'self_attn.o_proj', 'avg_loss': '0.0008', 'time': '0.2488'}\n",
+      "INFO - {'layer': 7, 'module': 'mlp.up_proj', 'avg_loss': '0.0667', 'time': '0.2496'}\n",
+      "INFO - {'layer': 7, 'module': 'mlp.gate_proj', 'avg_loss': '0.0965', 'time': '0.2494'}\n",
+      "INFO - {'layer': 7, 'module': 'mlp.down_proj', 'avg_loss': '0.0022', 'time': '0.6853'}\n",
+      "INFO - {'layer': 8, 'module': 'self_attn.k_proj', 'avg_loss': '0.0968', 'time': '0.2501'}\n",
+      "INFO - {'layer': 8, 'module': 'self_attn.v_proj', 'avg_loss': '0.0316', 'time': '0.2510'}\n",
+      "INFO - {'layer': 8, 'module': 'self_attn.q_proj', 'avg_loss': '0.0997', 'time': '0.2578'}\n",
+      "INFO - {'layer': 8, 'module': 'self_attn.o_proj', 'avg_loss': '0.0010', 'time': '0.2455'}\n",
+      "INFO - {'layer': 8, 'module': 'mlp.up_proj', 'avg_loss': '0.0680', 'time': '0.2488'}\n",
+      "INFO - {'layer': 8, 'module': 'mlp.gate_proj', 'avg_loss': '0.0904', 'time': '0.2472'}\n",
+      "INFO - {'layer': 8, 'module': 'mlp.down_proj', 'avg_loss': '0.0024', 'time': '0.6778'}\n",
+      "INFO - {'layer': 9, 'module': 'self_attn.k_proj', 'avg_loss': '0.0799', 'time': '0.2464'}\n",
+      "INFO - {'layer': 9, 'module': 'self_attn.v_proj', 'avg_loss': '0.0242', 'time': '0.2489'}\n",
+      "INFO - {'layer': 9, 'module': 'self_attn.q_proj', 'avg_loss': '0.0754', 'time': '0.2512'}\n",
+      "INFO - {'layer': 9, 'module': 'self_attn.o_proj', 'avg_loss': '0.0012', 'time': '0.2452'}\n",
+      "INFO - {'layer': 9, 'module': 'mlp.up_proj', 'avg_loss': '0.0668', 'time': '0.2462'}\n",
+      "INFO - {'layer': 9, 'module': 'mlp.gate_proj', 'avg_loss': '0.0844', 'time': '0.2466'}\n",
+      "INFO - {'layer': 9, 'module': 'mlp.down_proj', 'avg_loss': '0.0025', 'time': '0.6810'}\n",
+      "INFO - {'layer': 10, 'module': 'self_attn.k_proj', 'avg_loss': '0.0775', 'time': '0.2490'}\n",
+      "INFO - {'layer': 10, 'module': 'self_attn.v_proj', 'avg_loss': '0.0229', 'time': '0.2503'}\n",
+      "INFO - {'layer': 10, 'module': 'self_attn.q_proj', 'avg_loss': '0.0778', 'time': '0.2487'}\n",
+      "INFO - {'layer': 10, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2569'}\n",
+      "INFO - {'layer': 10, 'module': 'mlp.up_proj', 'avg_loss': '0.0604', 'time': '0.2461'}\n",
+      "INFO - {'layer': 10, 'module': 'mlp.gate_proj', 'avg_loss': '0.0760', 'time': '0.2470'}\n",
+      "INFO - {'layer': 10, 'module': 'mlp.down_proj', 'avg_loss': '0.0021', 'time': '0.6774'}\n",
+      "INFO - {'layer': 11, 'module': 'self_attn.k_proj', 'avg_loss': '0.0896', 'time': '0.2468'}\n",
+      "INFO - {'layer': 11, 'module': 'self_attn.v_proj', 'avg_loss': '0.0336', 'time': '0.2502'}\n",
+      "INFO - {'layer': 11, 'module': 'self_attn.q_proj', 'avg_loss': '0.0888', 'time': '0.2462'}\n",
+      "INFO - {'layer': 11, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2457'}\n",
+      "INFO - {'layer': 11, 'module': 'mlp.up_proj', 'avg_loss': '0.0564', 'time': '0.2464'}\n",
+      "INFO - {'layer': 11, 'module': 'mlp.gate_proj', 'avg_loss': '0.0658', 'time': '0.2463'}\n",
+      "INFO - {'layer': 11, 'module': 'mlp.down_proj', 'avg_loss': '0.0022', 'time': '0.6804'}\n",
+      "INFO - {'layer': 12, 'module': 'self_attn.k_proj', 'avg_loss': '0.0880', 'time': '0.2495'}\n",
+      "INFO - {'layer': 12, 'module': 'self_attn.v_proj', 'avg_loss': '0.0369', 'time': '0.2485'}\n",
+      "INFO - {'layer': 12, 'module': 'self_attn.q_proj', 'avg_loss': '0.0913', 'time': '0.2481'}\n",
+      "INFO - {'layer': 12, 'module': 'self_attn.o_proj', 'avg_loss': '0.0022', 'time': '0.2462'}\n",
+      "INFO - {'layer': 12, 'module': 'mlp.up_proj', 'avg_loss': '0.0626', 'time': '0.2482'}\n",
+      "INFO - {'layer': 12, 'module': 'mlp.gate_proj', 'avg_loss': '0.0757', 'time': '0.2524'}\n",
+      "INFO - {'layer': 12, 'module': 'mlp.down_proj', 'avg_loss': '0.0027', 'time': '0.6790'}\n",
+      "INFO - {'layer': 13, 'module': 'self_attn.k_proj', 'avg_loss': '0.0972', 'time': '0.2483'}\n",
+      "INFO - {'layer': 13, 'module': 'self_attn.v_proj', 'avg_loss': '0.0364', 'time': '0.2508'}\n",
+      "INFO - {'layer': 13, 'module': 'self_attn.q_proj', 'avg_loss': '0.0959', 'time': '0.2523'}\n",
+      "INFO - {'layer': 13, 'module': 'self_attn.o_proj', 'avg_loss': '0.0030', 'time': '0.2469'}\n",
+      "INFO - {'layer': 13, 'module': 'mlp.up_proj', 'avg_loss': '0.0606', 'time': '0.2467'}\n",
+      "INFO - {'layer': 13, 'module': 'mlp.gate_proj', 'avg_loss': '0.0698', 'time': '0.2461'}\n",
+      "INFO - {'layer': 13, 'module': 'mlp.down_proj', 'avg_loss': '0.0028', 'time': '0.6803'}\n",
+      "INFO - {'layer': 14, 'module': 'self_attn.k_proj', 'avg_loss': '0.0990', 'time': '0.2467'}\n",
+      "INFO - {'layer': 14, 'module': 'self_attn.v_proj', 'avg_loss': '0.0316', 'time': '0.2539'}\n",
+      "INFO - {'layer': 14, 'module': 'self_attn.q_proj', 'avg_loss': '0.0879', 'time': '0.2527'}\n",
+      "INFO - {'layer': 14, 'module': 'self_attn.o_proj', 'avg_loss': '0.0020', 'time': '0.2495'}\n",
+      "INFO - {'layer': 14, 'module': 'mlp.up_proj', 'avg_loss': '0.0633', 'time': '0.2473'}\n",
+      "INFO - {'layer': 14, 'module': 'mlp.gate_proj', 'avg_loss': '0.0739', 'time': '0.2488'}\n",
+      "INFO - {'layer': 14, 'module': 'mlp.down_proj', 'avg_loss': '0.0028', 'time': '0.6782'}\n",
+      "INFO - {'layer': 15, 'module': 'self_attn.k_proj', 'avg_loss': '0.0921', 'time': '0.2470'}\n",
+      "INFO - {'layer': 15, 'module': 'self_attn.v_proj', 'avg_loss': '0.0395', 'time': '0.2504'}\n",
+      "INFO - {'layer': 15, 'module': 'self_attn.q_proj', 'avg_loss': '0.0865', 'time': '0.2485'}\n",
+      "INFO - {'layer': 15, 'module': 'self_attn.o_proj', 'avg_loss': '0.0026', 'time': '0.2451'}\n",
+      "INFO - {'layer': 15, 'module': 'mlp.up_proj', 'avg_loss': '0.0699', 'time': '0.2465'}\n",
+      "INFO - {'layer': 15, 'module': 'mlp.gate_proj', 'avg_loss': '0.0791', 'time': '0.2461'}\n",
+      "INFO - {'layer': 15, 'module': 'mlp.down_proj', 'avg_loss': '0.0039', 'time': '0.6790'}\n",
+      "INFO - {'layer': 16, 'module': 'self_attn.k_proj', 'avg_loss': '0.0927', 'time': '0.2475'}\n",
+      "INFO - {'layer': 16, 'module': 'self_attn.v_proj', 'avg_loss': '0.0487', 'time': '0.2516'}\n",
+      "INFO - {'layer': 16, 'module': 'self_attn.q_proj', 'avg_loss': '0.0882', 'time': '0.2471'}\n",
+      "INFO - {'layer': 16, 'module': 'self_attn.o_proj', 'avg_loss': '0.0021', 'time': '0.2452'}\n",
+      "INFO - {'layer': 16, 'module': 'mlp.up_proj', 'avg_loss': '0.0842', 'time': '0.2463'}\n",
+      "INFO - {'layer': 16, 'module': 'mlp.gate_proj', 'avg_loss': '0.0926', 'time': '0.2461'}\n",
+      "INFO - {'layer': 16, 'module': 'mlp.down_proj', 'avg_loss': '0.0057', 'time': '0.6790'}\n",
+      "INFO - {'layer': 17, 'module': 'self_attn.k_proj', 'avg_loss': '0.1059', 'time': '0.2480'}\n",
+      "INFO - {'layer': 17, 'module': 'self_attn.v_proj', 'avg_loss': '0.0595', 'time': '0.2482'}\n",
+      "INFO - {'layer': 17, 'module': 'self_attn.q_proj', 'avg_loss': '0.0968', 'time': '0.2466'}\n",
+      "INFO - {'layer': 17, 'module': 'self_attn.o_proj', 'avg_loss': '0.0027', 'time': '0.2455'}\n",
+      "INFO - {'layer': 17, 'module': 'mlp.up_proj', 'avg_loss': '0.1018', 'time': '0.2471'}\n",
+      "INFO - {'layer': 17, 'module': 'mlp.gate_proj', 'avg_loss': '0.1146', 'time': '0.2473'}\n",
+      "INFO - {'layer': 17, 'module': 'mlp.down_proj', 'avg_loss': '0.0079', 'time': '0.6810'}\n",
+      "INFO - {'layer': 18, 'module': 'self_attn.k_proj', 'avg_loss': '0.1082', 'time': '0.2472'}\n",
+      "INFO - {'layer': 18, 'module': 'self_attn.v_proj', 'avg_loss': '0.0820', 'time': '0.2487'}\n",
+      "INFO - {'layer': 18, 'module': 'self_attn.q_proj', 'avg_loss': '0.1078', 'time': '0.2474'}\n",
+      "INFO - {'layer': 18, 'module': 'self_attn.o_proj', 'avg_loss': '0.0056', 'time': '0.2468'}\n",
+      "INFO - {'layer': 18, 'module': 'mlp.up_proj', 'avg_loss': '0.1236', 'time': '0.2455'}\n",
+      "INFO - {'layer': 18, 'module': 'mlp.gate_proj', 'avg_loss': '0.1284', 'time': '0.2460'}\n",
+      "INFO - {'layer': 18, 'module': 'mlp.down_proj', 'avg_loss': '0.0112', 'time': '0.6785'}\n",
+      "INFO - {'layer': 19, 'module': 'self_attn.k_proj', 'avg_loss': '0.1101', 'time': '0.2465'}\n",
+      "INFO - {'layer': 19, 'module': 'self_attn.v_proj', 'avg_loss': '0.1003', 'time': '0.2485'}\n",
+      "INFO - {'layer': 19, 'module': 'self_attn.q_proj', 'avg_loss': '0.0979', 'time': '0.2466'}\n",
+      "INFO - {'layer': 19, 'module': 'self_attn.o_proj', 'avg_loss': '0.0040', 'time': '0.2453'}\n",
+      "INFO - {'layer': 19, 'module': 'mlp.up_proj', 'avg_loss': '0.1472', 'time': '0.2464'}\n",
+      "INFO - {'layer': 19, 'module': 'mlp.gate_proj', 'avg_loss': '0.1505', 'time': '0.2499'}\n",
+      "INFO - {'layer': 19, 'module': 'mlp.down_proj', 'avg_loss': '0.0156', 'time': '0.6794'}\n",
+      "INFO - {'layer': 20, 'module': 'self_attn.k_proj', 'avg_loss': '0.1124', 'time': '0.2468'}\n",
+      "INFO - {'layer': 20, 'module': 'self_attn.v_proj', 'avg_loss': '0.1115', 'time': '0.2459'}\n",
+      "INFO - {'layer': 20, 'module': 'self_attn.q_proj', 'avg_loss': '0.1015', 'time': '0.2484'}\n",
+      "INFO - {'layer': 20, 'module': 'self_attn.o_proj', 'avg_loss': '0.0070', 'time': '0.2453'}\n",
+      "INFO - {'layer': 20, 'module': 'mlp.up_proj', 'avg_loss': '0.1550', 'time': '0.2450'}\n",
+      "INFO - {'layer': 20, 'module': 'mlp.gate_proj', 'avg_loss': '0.1512', 'time': '0.2451'}\n",
+      "INFO - {'layer': 20, 'module': 'mlp.down_proj', 'avg_loss': '0.0180', 'time': '0.6781'}\n",
+      "INFO - {'layer': 21, 'module': 'self_attn.k_proj', 'avg_loss': '0.1197', 'time': '0.2465'}\n",
+      "INFO - {'layer': 21, 'module': 'self_attn.v_proj', 'avg_loss': '0.1370', 'time': '0.2484'}\n",
+      "INFO - {'layer': 21, 'module': 'self_attn.q_proj', 'avg_loss': '0.1034', 'time': '0.2461'}\n",
+      "INFO - {'layer': 21, 'module': 'self_attn.o_proj', 'avg_loss': '0.0058', 'time': '0.2492'}\n",
+      "INFO - {'layer': 21, 'module': 'mlp.up_proj', 'avg_loss': '0.1678', 'time': '0.2464'}\n",
+      "INFO - {'layer': 21, 'module': 'mlp.gate_proj', 'avg_loss': '0.1542', 'time': '0.2494'}\n",
+      "INFO - {'layer': 21, 'module': 'mlp.down_proj', 'avg_loss': '0.0229', 'time': '0.6776'}\n",
+      "INFO - {'layer': 22, 'module': 'self_attn.k_proj', 'avg_loss': '0.1123', 'time': '0.2468'}\n",
+      "INFO - {'layer': 22, 'module': 'self_attn.v_proj', 'avg_loss': '0.1440', 'time': '0.2480'}\n",
+      "INFO - {'layer': 22, 'module': 'self_attn.q_proj', 'avg_loss': '0.1030', 'time': '0.2464'}\n",
+      "INFO - {'layer': 22, 'module': 'self_attn.o_proj', 'avg_loss': '0.0116', 'time': '0.2450'}\n",
+      "INFO - {'layer': 22, 'module': 'mlp.up_proj', 'avg_loss': '0.1743', 'time': '0.2452'}\n",
+      "INFO - {'layer': 22, 'module': 'mlp.gate_proj', 'avg_loss': '0.1545', 'time': '0.2481'}\n",
+      "INFO - {'layer': 22, 'module': 'mlp.down_proj', 'avg_loss': '0.0335', 'time': '0.6765'}\n",
+      "INFO - {'layer': 23, 'module': 'self_attn.k_proj', 'avg_loss': '0.1069', 'time': '0.2518'}\n",
+      "INFO - {'layer': 23, 'module': 'self_attn.v_proj', 'avg_loss': '0.1511', 'time': '0.2485'}\n",
+      "INFO - {'layer': 23, 'module': 'self_attn.q_proj', 'avg_loss': '0.1324', 'time': '0.2463'}\n",
+      "INFO - {'layer': 23, 'module': 'self_attn.o_proj', 'avg_loss': '0.0338', 'time': '0.2454'}\n",
+      "INFO - {'layer': 23, 'module': 'mlp.up_proj', 'avg_loss': '0.1691', 'time': '0.2470'}\n",
+      "INFO - {'layer': 23, 'module': 'mlp.gate_proj', 'avg_loss': '0.1514', 'time': '0.2481'}\n",
+      "INFO - {'layer': 23, 'module': 'mlp.down_proj', 'avg_loss': '0.1119', 'time': '0.6800'}\n",
+      "INFO - {'layer': 24, 'module': 'self_attn.k_proj', 'avg_loss': '0.0888', 'time': '0.2469'}\n",
+      "INFO - {'layer': 24, 'module': 'self_attn.v_proj', 'avg_loss': '0.1012', 'time': '0.2473'}\n",
+      "INFO - {'layer': 24, 'module': 'self_attn.q_proj', 'avg_loss': '0.0848', 'time': '0.2462'}\n",
+      "INFO - {'layer': 24, 'module': 'self_attn.o_proj', 'avg_loss': '0.0100', 'time': '0.2472'}\n",
+      "INFO - {'layer': 24, 'module': 'mlp.up_proj', 'avg_loss': '0.2124', 'time': '0.2458'}\n",
+      "INFO - {'layer': 24, 'module': 'mlp.gate_proj', 'avg_loss': '0.1940', 'time': '0.2455'}\n",
+      "INFO - {'layer': 24, 'module': 'mlp.down_proj', 'avg_loss': '0.1381', 'time': '0.6771'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "# only use 0 gpu_device\n",
+    "model = GPTQModel.from_pretrained(pretrained_model_dir, quantize_config, trust_remote_code=True,\n",
+    "                                  device_map=\"balanced_low_0\") \n",
+    "refresh_log()\n",
+    "\n",
+    "logging.info(f\"start quantize\")\n",
+    "# quantize model, the examples should be list of dict whose keys can only be \"input_ids\" and \"attention_mask\"\n",
+    "# cache_examples_on_gpu=False\n",
+    "quant_log, quantizers, force_layer_back_to_cpu, device_map, forward_pass_use_cache = model.quantize(examples, batch_size=1, calibration_enable_gpu_cache=False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "be5f1ad7-cca6-48a2-a62a-1c4adb50f99f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['forward_pass_use_cache.joblib']"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# save all varabiles\n",
+    "from joblib import dump, load\n",
+    "dump(model, 'model.joblib')\n",
+    "dump(quant_log, 'quant_log.joblib')\n",
+    "dump(quantizers, 'quantizers.joblib')\n",
+    "dump(force_layer_back_to_cpu, 'force_layer_back_to_cpu.joblib')\n",
+    "dump(device_map, 'device_map.joblib')\n",
+    "dump(forward_pass_use_cache, 'forward_pass_use_cache.joblib')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5462d730-1d6d-4654-87c3-a90f7c334c98",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from joblib import dump, load\n",
+    "model = load('model.joblib')\n",
+    "quant_log = load('quant_log.joblib')\n",
+    "quantizers = load('quantizers.joblib')\n",
+    "force_layer_back_to_cpu = load('force_layer_back_to_cpu.joblib')\n",
+    "device_map = load('device_map.joblib')\n",
+    "forward_pass_use_cache = load('forward_pass_use_cache.joblib')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "4b4a0cf6-d413-40a9-a3e8-4ad0945c5095",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO 2024-07-17 20:27:58.837 [61754-MainThread] (<module>@3912912720.py:001) start pack\n",
+      "INFO 2024-07-17 20:27:58.838 [61754-MainThread] (select_quant_linear@importer.py:051) Auto choose the fastest one based on quant model compatibility: <class 'gptqmodel.nn_modules.qlinear.qlinear_tritonv2.TritonV2QuantLinear'>\n",
+      "INFO - Packing model...\n",
+      "INFO 2024-07-17 20:27:58.841 [61754-MainThread] (pack_model@model.py:278) Packing model...\n",
+      "INFO 2024-07-17 20:27:58.842 [61754-MainThread] (select_quant_linear@importer.py:051) Auto choose the fastest one based on quant model compatibility: <class 'gptqmodel.nn_modules.qlinear.qlinear_tritonv2.TritonV2QuantLinear'>\n",
+      "  0%|          | 0/168 [00:00<?, ?it/s]INFO 2024-07-17 20:28:04.998 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.0.self_attn.o_proj to None\n",
+      "  1%|          | 1/168 [00:05<15:54,  5.71s/it]INFO 2024-07-17 20:28:05.052 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.2.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:05.090 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.1.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.115 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.5.self_attn.k_proj to None\n",
+      "  2%|▏         | 4/168 [00:05<03:02,  1.11s/it]INFO 2024-07-17 20:28:05.157 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.4.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.164 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.4.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:05.179 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.2.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.182 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.0.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.207 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.2.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.229 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.0.self_attn.v_proj to None\n",
+      "  6%|▌         | 10/168 [00:05<00:55,  2.87it/s]INFO 2024-07-17 20:28:05.267 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.3.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.292 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.1.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:05.298 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.1.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:05.306 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.0.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.317 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.3.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.345 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.6.self_attn.o_proj to None\n",
+      " 10%|▉         | 16/168 [00:06<00:28,  5.42it/s]INFO 2024-07-17 20:28:05.367 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.8.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.372 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.7.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.377 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.8.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:05.382 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.8.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:05.399 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.3.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:05.409 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.2.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:05.413 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.1.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.429 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.4.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.436 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.5.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.441 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.6.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.443 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.0.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.446 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.5.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:05.456 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.7.self_attn.q_proj to None\n",
+      " 17%|█▋        | 29/168 [00:06<00:10, 12.89it/s]INFO 2024-07-17 20:28:05.476 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.4.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:05.486 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.6.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.492 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.5.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.503 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.2.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.513 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.1.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.525 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.8.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.527 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.7.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:05.530 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.1.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.559 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.4.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.581 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.6.self_attn.v_proj to None\n",
+      " 23%|██▎       | 39/168 [00:06<00:06, 19.31it/s]INFO 2024-07-17 20:28:05.595 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.7.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:05.603 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.3.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.605 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.2.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.606 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.9.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:05.621 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.7.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.642 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.7.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.670 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.6.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.673 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.4.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.689 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.5.self_attn.v_proj to None\n",
+      " 29%|██▊       | 48/168 [00:06<00:04, 26.28it/s]INFO 2024-07-17 20:28:05.697 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.3.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:05.705 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.5.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.711 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.8.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:05.737 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.3.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.753 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.0.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:05.780 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.6.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:05.844 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.8.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:10.350 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.9.self_attn.k_proj to None\n",
+      " 33%|███▎      | 56/168 [00:11<00:22,  4.94it/s]INFO 2024-07-17 20:28:10.377 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.9.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:10.453 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.9.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:10.465 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.9.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:10.531 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.10.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:10.558 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.9.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:10.565 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.11.self_attn.o_proj to None\n",
+      " 37%|███▋      | 62/168 [00:11<00:17,  6.20it/s]INFO 2024-07-17 20:28:10.581 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.10.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:10.606 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.10.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:10.625 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.10.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:10.630 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.10.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:10.645 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.10.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:10.667 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.11.self_attn.v_proj to None\n",
+      " 40%|████      | 68/168 [00:11<00:12,  8.06it/s]INFO 2024-07-17 20:28:10.717 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.11.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:10.768 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.12.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:10.784 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.12.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:10.817 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.11.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:10.825 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.12.mlp.gate_proj to None\n",
+      " 43%|████▎     | 73/168 [00:11<00:09,  9.76it/s]INFO 2024-07-17 20:28:10.839 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.11.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:10.852 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.12.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:10.887 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.13.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:10.891 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.11.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:10.915 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.13.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:10.946 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.12.mlp.up_proj to None\n",
+      " 47%|████▋     | 79/168 [00:11<00:07, 12.71it/s]INFO 2024-07-17 20:28:10.973 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.12.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:11.012 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.13.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:11.023 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.13.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:11.052 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.13.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:11.066 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.14.self_attn.q_proj to None\n",
+      " 50%|█████     | 84/168 [00:11<00:05, 15.35it/s]INFO 2024-07-17 20:28:11.087 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.14.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:11.090 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.14.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:11.176 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.14.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:11.184 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.14.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:11.219 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.13.mlp.up_proj to None\n",
+      " 53%|█████▎    | 89/168 [00:11<00:04, 18.01it/s]INFO 2024-07-17 20:28:11.242 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.15.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:11.264 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.14.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:11.283 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.16.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:11.313 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.16.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:11.318 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.15.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:11.333 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.15.self_attn.q_proj to None\n",
+      " 57%|█████▋    | 95/168 [00:12<00:03, 22.71it/s]INFO 2024-07-17 20:28:11.359 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.15.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:11.379 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.16.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:11.398 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.15.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:11.418 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.16.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:11.435 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.15.self_attn.k_proj to None\n",
+      " 60%|█████▉    | 100/168 [00:12<00:02, 26.56it/s]INFO 2024-07-17 20:28:11.482 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.16.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:11.520 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.16.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:14.653 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.7.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:14.674 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.3.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:14.692 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.1.mlp.down_proj to None\n",
+      " 62%|██████▎   | 105/168 [00:15<00:13,  4.78it/s]INFO 2024-07-17 20:28:14.723 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.5.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:14.744 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.6.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:14.770 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.4.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:14.833 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.2.mlp.down_proj to None\n",
+      " 65%|██████▍   | 109/168 [00:15<00:09,  5.98it/s]INFO 2024-07-17 20:28:14.875 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.8.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:14.901 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.0.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:15.179 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.17.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:15.234 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.17.self_attn.k_proj to None\n",
+      " 67%|██████▋   | 113/168 [00:15<00:08,  6.67it/s]INFO 2024-07-17 20:28:15.361 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.17.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:15.402 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.18.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:15.411 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.18.self_attn.q_proj to None\n",
+      " 69%|██████▉   | 116/168 [00:16<00:06,  7.67it/s]INFO 2024-07-17 20:28:15.419 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.18.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:15.423 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.17.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:15.452 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.17.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:15.459 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.18.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:15.473 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.18.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:15.522 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.17.mlp.gate_proj to None\n",
+      " 73%|███████▎  | 122/168 [00:16<00:04, 11.41it/s]INFO 2024-07-17 20:28:15.560 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.19.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:15.575 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.18.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:15.582 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.19.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:15.599 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.20.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:15.630 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.20.self_attn.q_proj to None\n",
+      " 76%|███████▌  | 127/168 [00:16<00:02, 14.91it/s]INFO 2024-07-17 20:28:15.634 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.19.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:15.646 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.19.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:15.648 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.20.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:15.667 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.20.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:15.678 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.19.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:15.691 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.19.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:15.704 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.21.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:15.731 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.20.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:15.777 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.22.mlp.gate_proj to None\n",
+      " 81%|████████  | 136/168 [00:16<00:01, 22.53it/s]INFO 2024-07-17 20:28:15.790 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.22.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:15.804 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.20.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:15.816 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.21.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:15.830 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.23.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:15.832 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.21.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:15.846 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.21.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:15.851 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.21.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:15.857 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.22.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:15.868 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.22.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:15.873 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.22.self_attn.k_proj to None\n",
+      "INFO 2024-07-17 20:28:15.885 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.21.self_attn.v_proj to None\n",
+      " 88%|████████▊ | 147/168 [00:16<00:00, 34.18it/s]INFO 2024-07-17 20:28:15.895 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.23.self_attn.o_proj to None\n",
+      "INFO 2024-07-17 20:28:15.903 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.23.self_attn.v_proj to None\n",
+      "INFO 2024-07-17 20:28:15.916 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.22.mlp.up_proj to None\n",
+      "INFO 2024-07-17 20:28:15.924 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.23.self_attn.q_proj to None\n",
+      "INFO 2024-07-17 20:28:15.943 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.23.mlp.gate_proj to None\n",
+      "INFO 2024-07-17 20:28:16.341 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.9.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.455 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.11.mlp.down_proj to None\n",
+      " 92%|█████████▏| 154/168 [00:17<00:00, 22.68it/s]INFO 2024-07-17 20:28:16.462 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.10.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.483 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.12.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.490 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.13.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.515 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.15.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.528 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.14.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.552 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.16.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.613 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.23.mlp.up_proj to None\n",
+      " 96%|█████████▌| 161/168 [00:17<00:00, 26.42it/s]INFO 2024-07-17 20:28:16.854 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.17.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.882 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.19.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.971 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.21.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.992 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.22.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:16.992 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.18.mlp.down_proj to None\n",
+      " 99%|█████████▉| 166/168 [00:17<00:00, 21.48it/s]INFO 2024-07-17 20:28:17.000 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.20.mlp.down_proj to None\n",
+      "INFO 2024-07-17 20:28:17.034 [61754-MainThread] (pack_model@model.py:299) Processed model.layers.23.mlp.down_proj to None\n",
+      "100%|██████████| 168/168 [00:17<00:00,  9.47it/s]\n",
+      "INFO - Model packed.\n",
+      "INFO 2024-07-17 20:28:17.039 [61754-MainThread] (pack_model@model.py:301) Model packed.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'layer': 1,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0039',\n",
+       "  'time': '0.2490'},\n",
+       " {'layer': 1,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0001',\n",
+       "  'time': '0.2508'},\n",
+       " {'layer': 1,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0058',\n",
+       "  'time': '0.2535'},\n",
+       " {'layer': 1,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0000',\n",
+       "  'time': '0.2501'},\n",
+       " {'layer': 1, 'module': 'mlp.up_proj', 'avg_loss': '0.0183', 'time': '0.2542'},\n",
+       " {'layer': 1,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0249',\n",
+       "  'time': '0.2524'},\n",
+       " {'layer': 1,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0009',\n",
+       "  'time': '0.7110'},\n",
+       " {'layer': 2,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0487',\n",
+       "  'time': '0.2542'},\n",
+       " {'layer': 2,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0056',\n",
+       "  'time': '0.2556'},\n",
+       " {'layer': 2,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0434',\n",
+       "  'time': '0.2489'},\n",
+       " {'layer': 2,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0001',\n",
+       "  'time': '0.2569'},\n",
+       " {'layer': 2, 'module': 'mlp.up_proj', 'avg_loss': '0.0265', 'time': '0.2470'},\n",
+       " {'layer': 2,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0431',\n",
+       "  'time': '0.2478'},\n",
+       " {'layer': 2,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0006',\n",
+       "  'time': '0.6856'},\n",
+       " {'layer': 3,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0950',\n",
+       "  'time': '0.2479'},\n",
+       " {'layer': 3,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0140',\n",
+       "  'time': '0.2482'},\n",
+       " {'layer': 3,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0834',\n",
+       "  'time': '0.2478'},\n",
+       " {'layer': 3,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0002',\n",
+       "  'time': '0.2489'},\n",
+       " {'layer': 3, 'module': 'mlp.up_proj', 'avg_loss': '0.0338', 'time': '0.2457'},\n",
+       " {'layer': 3,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0550',\n",
+       "  'time': '0.2464'},\n",
+       " {'layer': 3,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0009',\n",
+       "  'time': '0.6789'},\n",
+       " {'layer': 4,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0903',\n",
+       "  'time': '0.2479'},\n",
+       " {'layer': 4,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0177',\n",
+       "  'time': '0.2490'},\n",
+       " {'layer': 4,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0796',\n",
+       "  'time': '0.2491'},\n",
+       " {'layer': 4,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0003',\n",
+       "  'time': '0.2473'},\n",
+       " {'layer': 4, 'module': 'mlp.up_proj', 'avg_loss': '0.0439', 'time': '0.2482'},\n",
+       " {'layer': 4,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0609',\n",
+       "  'time': '0.2463'},\n",
+       " {'layer': 4,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0013',\n",
+       "  'time': '0.6797'},\n",
+       " {'layer': 5,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0705',\n",
+       "  'time': '0.2469'},\n",
+       " {'layer': 5,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0124',\n",
+       "  'time': '0.2492'},\n",
+       " {'layer': 5,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0630',\n",
+       "  'time': '0.2472'},\n",
+       " {'layer': 5,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0004',\n",
+       "  'time': '0.2458'},\n",
+       " {'layer': 5, 'module': 'mlp.up_proj', 'avg_loss': '0.0506', 'time': '0.2469'},\n",
+       " {'layer': 5,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0733',\n",
+       "  'time': '0.2465'},\n",
+       " {'layer': 5,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0014',\n",
+       "  'time': '0.6803'},\n",
+       " {'layer': 6,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0940',\n",
+       "  'time': '0.2497'},\n",
+       " {'layer': 6,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0180',\n",
+       "  'time': '0.2521'},\n",
+       " {'layer': 6,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0856',\n",
+       "  'time': '0.2488'},\n",
+       " {'layer': 6,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0007',\n",
+       "  'time': '0.2459'},\n",
+       " {'layer': 6, 'module': 'mlp.up_proj', 'avg_loss': '0.0614', 'time': '0.2470'},\n",
+       " {'layer': 6,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0813',\n",
+       "  'time': '0.2456'},\n",
+       " {'layer': 6,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0075',\n",
+       "  'time': '0.6766'},\n",
+       " {'layer': 7,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0902',\n",
+       "  'time': '0.2492'},\n",
+       " {'layer': 7,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0270',\n",
+       "  'time': '0.2521'},\n",
+       " {'layer': 7,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0869',\n",
+       "  'time': '0.2503'},\n",
+       " {'layer': 7,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0008',\n",
+       "  'time': '0.2488'},\n",
+       " {'layer': 7, 'module': 'mlp.up_proj', 'avg_loss': '0.0667', 'time': '0.2496'},\n",
+       " {'layer': 7,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0965',\n",
+       "  'time': '0.2494'},\n",
+       " {'layer': 7,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0022',\n",
+       "  'time': '0.6853'},\n",
+       " {'layer': 8,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0968',\n",
+       "  'time': '0.2501'},\n",
+       " {'layer': 8,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0316',\n",
+       "  'time': '0.2510'},\n",
+       " {'layer': 8,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0997',\n",
+       "  'time': '0.2578'},\n",
+       " {'layer': 8,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0010',\n",
+       "  'time': '0.2455'},\n",
+       " {'layer': 8, 'module': 'mlp.up_proj', 'avg_loss': '0.0680', 'time': '0.2488'},\n",
+       " {'layer': 8,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0904',\n",
+       "  'time': '0.2472'},\n",
+       " {'layer': 8,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0024',\n",
+       "  'time': '0.6778'},\n",
+       " {'layer': 9,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0799',\n",
+       "  'time': '0.2464'},\n",
+       " {'layer': 9,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0242',\n",
+       "  'time': '0.2489'},\n",
+       " {'layer': 9,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0754',\n",
+       "  'time': '0.2512'},\n",
+       " {'layer': 9,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0012',\n",
+       "  'time': '0.2452'},\n",
+       " {'layer': 9, 'module': 'mlp.up_proj', 'avg_loss': '0.0668', 'time': '0.2462'},\n",
+       " {'layer': 9,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0844',\n",
+       "  'time': '0.2466'},\n",
+       " {'layer': 9,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0025',\n",
+       "  'time': '0.6810'},\n",
+       " {'layer': 10,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0775',\n",
+       "  'time': '0.2490'},\n",
+       " {'layer': 10,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0229',\n",
+       "  'time': '0.2503'},\n",
+       " {'layer': 10,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0778',\n",
+       "  'time': '0.2487'},\n",
+       " {'layer': 10,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0020',\n",
+       "  'time': '0.2569'},\n",
+       " {'layer': 10,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.0604',\n",
+       "  'time': '0.2461'},\n",
+       " {'layer': 10,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0760',\n",
+       "  'time': '0.2470'},\n",
+       " {'layer': 10,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0021',\n",
+       "  'time': '0.6774'},\n",
+       " {'layer': 11,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0896',\n",
+       "  'time': '0.2468'},\n",
+       " {'layer': 11,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0336',\n",
+       "  'time': '0.2502'},\n",
+       " {'layer': 11,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0888',\n",
+       "  'time': '0.2462'},\n",
+       " {'layer': 11,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0020',\n",
+       "  'time': '0.2457'},\n",
+       " {'layer': 11,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.0564',\n",
+       "  'time': '0.2464'},\n",
+       " {'layer': 11,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0658',\n",
+       "  'time': '0.2463'},\n",
+       " {'layer': 11,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0022',\n",
+       "  'time': '0.6804'},\n",
+       " {'layer': 12,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0880',\n",
+       "  'time': '0.2495'},\n",
+       " {'layer': 12,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0369',\n",
+       "  'time': '0.2485'},\n",
+       " {'layer': 12,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0913',\n",
+       "  'time': '0.2481'},\n",
+       " {'layer': 12,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0022',\n",
+       "  'time': '0.2462'},\n",
+       " {'layer': 12,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.0626',\n",
+       "  'time': '0.2482'},\n",
+       " {'layer': 12,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0757',\n",
+       "  'time': '0.2524'},\n",
+       " {'layer': 12,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0027',\n",
+       "  'time': '0.6790'},\n",
+       " {'layer': 13,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0972',\n",
+       "  'time': '0.2483'},\n",
+       " {'layer': 13,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0364',\n",
+       "  'time': '0.2508'},\n",
+       " {'layer': 13,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0959',\n",
+       "  'time': '0.2523'},\n",
+       " {'layer': 13,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0030',\n",
+       "  'time': '0.2469'},\n",
+       " {'layer': 13,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.0606',\n",
+       "  'time': '0.2467'},\n",
+       " {'layer': 13,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0698',\n",
+       "  'time': '0.2461'},\n",
+       " {'layer': 13,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0028',\n",
+       "  'time': '0.6803'},\n",
+       " {'layer': 14,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0990',\n",
+       "  'time': '0.2467'},\n",
+       " {'layer': 14,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0316',\n",
+       "  'time': '0.2539'},\n",
+       " {'layer': 14,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0879',\n",
+       "  'time': '0.2527'},\n",
+       " {'layer': 14,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0020',\n",
+       "  'time': '0.2495'},\n",
+       " {'layer': 14,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.0633',\n",
+       "  'time': '0.2473'},\n",
+       " {'layer': 14,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0739',\n",
+       "  'time': '0.2488'},\n",
+       " {'layer': 14,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0028',\n",
+       "  'time': '0.6782'},\n",
+       " {'layer': 15,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0921',\n",
+       "  'time': '0.2470'},\n",
+       " {'layer': 15,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0395',\n",
+       "  'time': '0.2504'},\n",
+       " {'layer': 15,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0865',\n",
+       "  'time': '0.2485'},\n",
+       " {'layer': 15,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0026',\n",
+       "  'time': '0.2451'},\n",
+       " {'layer': 15,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.0699',\n",
+       "  'time': '0.2465'},\n",
+       " {'layer': 15,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0791',\n",
+       "  'time': '0.2461'},\n",
+       " {'layer': 15,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0039',\n",
+       "  'time': '0.6790'},\n",
+       " {'layer': 16,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0927',\n",
+       "  'time': '0.2475'},\n",
+       " {'layer': 16,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0487',\n",
+       "  'time': '0.2516'},\n",
+       " {'layer': 16,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0882',\n",
+       "  'time': '0.2471'},\n",
+       " {'layer': 16,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0021',\n",
+       "  'time': '0.2452'},\n",
+       " {'layer': 16,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.0842',\n",
+       "  'time': '0.2463'},\n",
+       " {'layer': 16,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.0926',\n",
+       "  'time': '0.2461'},\n",
+       " {'layer': 16,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0057',\n",
+       "  'time': '0.6790'},\n",
+       " {'layer': 17,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.1059',\n",
+       "  'time': '0.2480'},\n",
+       " {'layer': 17,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0595',\n",
+       "  'time': '0.2482'},\n",
+       " {'layer': 17,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0968',\n",
+       "  'time': '0.2466'},\n",
+       " {'layer': 17,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0027',\n",
+       "  'time': '0.2455'},\n",
+       " {'layer': 17,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.1018',\n",
+       "  'time': '0.2471'},\n",
+       " {'layer': 17,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1146',\n",
+       "  'time': '0.2473'},\n",
+       " {'layer': 17,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0079',\n",
+       "  'time': '0.6810'},\n",
+       " {'layer': 18,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.1082',\n",
+       "  'time': '0.2472'},\n",
+       " {'layer': 18,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.0820',\n",
+       "  'time': '0.2487'},\n",
+       " {'layer': 18,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.1078',\n",
+       "  'time': '0.2474'},\n",
+       " {'layer': 18,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0056',\n",
+       "  'time': '0.2468'},\n",
+       " {'layer': 18,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.1236',\n",
+       "  'time': '0.2455'},\n",
+       " {'layer': 18,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1284',\n",
+       "  'time': '0.2460'},\n",
+       " {'layer': 18,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0112',\n",
+       "  'time': '0.6785'},\n",
+       " {'layer': 19,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.1101',\n",
+       "  'time': '0.2465'},\n",
+       " {'layer': 19,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.1003',\n",
+       "  'time': '0.2485'},\n",
+       " {'layer': 19,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0979',\n",
+       "  'time': '0.2466'},\n",
+       " {'layer': 19,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0040',\n",
+       "  'time': '0.2453'},\n",
+       " {'layer': 19,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.1472',\n",
+       "  'time': '0.2464'},\n",
+       " {'layer': 19,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1505',\n",
+       "  'time': '0.2499'},\n",
+       " {'layer': 19,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0156',\n",
+       "  'time': '0.6794'},\n",
+       " {'layer': 20,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.1124',\n",
+       "  'time': '0.2468'},\n",
+       " {'layer': 20,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.1115',\n",
+       "  'time': '0.2459'},\n",
+       " {'layer': 20,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.1015',\n",
+       "  'time': '0.2484'},\n",
+       " {'layer': 20,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0070',\n",
+       "  'time': '0.2453'},\n",
+       " {'layer': 20,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.1550',\n",
+       "  'time': '0.2450'},\n",
+       " {'layer': 20,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1512',\n",
+       "  'time': '0.2451'},\n",
+       " {'layer': 20,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0180',\n",
+       "  'time': '0.6781'},\n",
+       " {'layer': 21,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.1197',\n",
+       "  'time': '0.2465'},\n",
+       " {'layer': 21,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.1370',\n",
+       "  'time': '0.2484'},\n",
+       " {'layer': 21,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.1034',\n",
+       "  'time': '0.2461'},\n",
+       " {'layer': 21,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0058',\n",
+       "  'time': '0.2492'},\n",
+       " {'layer': 21,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.1678',\n",
+       "  'time': '0.2464'},\n",
+       " {'layer': 21,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1542',\n",
+       "  'time': '0.2494'},\n",
+       " {'layer': 21,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0229',\n",
+       "  'time': '0.6776'},\n",
+       " {'layer': 22,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.1123',\n",
+       "  'time': '0.2468'},\n",
+       " {'layer': 22,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.1440',\n",
+       "  'time': '0.2480'},\n",
+       " {'layer': 22,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.1030',\n",
+       "  'time': '0.2464'},\n",
+       " {'layer': 22,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0116',\n",
+       "  'time': '0.2450'},\n",
+       " {'layer': 22,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.1743',\n",
+       "  'time': '0.2452'},\n",
+       " {'layer': 22,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1545',\n",
+       "  'time': '0.2481'},\n",
+       " {'layer': 22,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.0335',\n",
+       "  'time': '0.6765'},\n",
+       " {'layer': 23,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.1069',\n",
+       "  'time': '0.2518'},\n",
+       " {'layer': 23,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.1511',\n",
+       "  'time': '0.2485'},\n",
+       " {'layer': 23,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.1324',\n",
+       "  'time': '0.2463'},\n",
+       " {'layer': 23,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0338',\n",
+       "  'time': '0.2454'},\n",
+       " {'layer': 23,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.1691',\n",
+       "  'time': '0.2470'},\n",
+       " {'layer': 23,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1514',\n",
+       "  'time': '0.2481'},\n",
+       " {'layer': 23,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.1119',\n",
+       "  'time': '0.6800'},\n",
+       " {'layer': 24,\n",
+       "  'module': 'self_attn.k_proj',\n",
+       "  'avg_loss': '0.0888',\n",
+       "  'time': '0.2469'},\n",
+       " {'layer': 24,\n",
+       "  'module': 'self_attn.v_proj',\n",
+       "  'avg_loss': '0.1012',\n",
+       "  'time': '0.2473'},\n",
+       " {'layer': 24,\n",
+       "  'module': 'self_attn.q_proj',\n",
+       "  'avg_loss': '0.0848',\n",
+       "  'time': '0.2462'},\n",
+       " {'layer': 24,\n",
+       "  'module': 'self_attn.o_proj',\n",
+       "  'avg_loss': '0.0100',\n",
+       "  'time': '0.2472'},\n",
+       " {'layer': 24,\n",
+       "  'module': 'mlp.up_proj',\n",
+       "  'avg_loss': '0.2124',\n",
+       "  'time': '0.2458'},\n",
+       " {'layer': 24,\n",
+       "  'module': 'mlp.gate_proj',\n",
+       "  'avg_loss': '0.1940',\n",
+       "  'time': '0.2455'},\n",
+       " {'layer': 24,\n",
+       "  'module': 'mlp.down_proj',\n",
+       "  'avg_loss': '0.1381',\n",
+       "  'time': '0.6771'}]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "logging.info(f\"start pack\")\n",
+    "model.pack(quant_log, quantizers, force_layer_back_to_cpu, None, forward_pass_use_cache)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "85a3f4a5-046f-45db-a4e7-5601985df7e7",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment.  If you were attempting to deepcopy a module, this may be because of a torch.nn.utils.weight_norm usage, see https://github.com/pytorch/pytorch/pull/103001",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# save quantized model using safetensors\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave_quantized\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquantized_model_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muse_safetensors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/aigc-nas02/workspace/online/llm_infer/GPTQModel/gptqmodel/models/base.py:624\u001b[0m, in \u001b[0;36mBaseGPTQModel.save_quantized\u001b[0;34m(self, save_dir, safetensors_metadata, use_safetensors, max_shard_size, model_base_name)\u001b[0m\n\u001b[1;32m    620\u001b[0m \u001b[38;5;66;03m# internal is always gptq v2 but allow users to pass gptq (v1) via config\u001b[39;00m\n\u001b[1;32m    621\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m quantize_config\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;241m==\u001b[39m FORMAT\u001b[38;5;241m.\u001b[39mGPTQ:\n\u001b[1;32m    622\u001b[0m     \u001b[38;5;66;03m# Model qzeros may be edited in place.\u001b[39;00m\n\u001b[1;32m    623\u001b[0m     \u001b[38;5;66;03m# TODO: avoid inplace modification of the weights\u001b[39;00m\n\u001b[0;32m--> 624\u001b[0m     model \u001b[38;5;241m=\u001b[39m \u001b[43mcopy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    625\u001b[0m     model \u001b[38;5;241m=\u001b[39m convert_gptq_v2_to_v1_format(\n\u001b[1;32m    626\u001b[0m         model, quantize_config\u001b[38;5;241m=\u001b[39mquantize_config, qlinear_kernel\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqlinear_kernel\n\u001b[1;32m    627\u001b[0m     )\n\u001b[1;32m    629\u001b[0m model\u001b[38;5;241m.\u001b[39mto(CPU)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (1 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (1 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (7 times), _deepcopy_dict at line 230 (3 times), _reconstruct at line 270 (3 times), _reconstruct at line 296 (3 times), deepcopy at line 146 (3 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:153\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    151\u001b[0m copier \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(x, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__deepcopy__\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    152\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 153\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    154\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    155\u001b[0m     reductor \u001b[38;5;241m=\u001b[39m dispatch_table\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/site-packages/torch/_tensor.py:86\u001b[0m, in \u001b[0;36mTensor.__deepcopy__\u001b[0;34m(self, memo)\u001b[0m\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(Tensor\u001b[38;5;241m.\u001b[39m__deepcopy__, (\u001b[38;5;28mself\u001b[39m,), \u001b[38;5;28mself\u001b[39m, memo)\n\u001b[1;32m     85\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_leaf:\n\u001b[0;32m---> 86\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m     87\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOnly Tensors created explicitly by the user \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     88\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m(graph leaves) support the deepcopy protocol at the moment.  \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     89\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIf you were attempting to deepcopy a module, this may be because \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     90\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mof a torch.nn.utils.weight_norm usage, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     91\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msee https://github.com/pytorch/pytorch/pull/103001\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     92\u001b[0m     )\n\u001b[1;32m     93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mid\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;129;01min\u001b[39;00m memo:\n\u001b[1;32m     94\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m memo[\u001b[38;5;28mid\u001b[39m(\u001b[38;5;28mself\u001b[39m)]\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment.  If you were attempting to deepcopy a module, this may be because of a torch.nn.utils.weight_norm usage, see https://github.com/pytorch/pytorch/pull/103001"
+     ]
+    }
+   ],
+   "source": [
+    "# save quantized model using safetensors\n",
+    "model.save_quantized(quantized_model_dir, use_safetensors=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "908f5642-526f-45b3-9324-8cb14b037b6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment.  If you were attempting to deepcopy a module, this may be because of a torch.nn.utils.weight_norm usage, see https://github.com/pytorch/pytorch/pull/103001",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[27], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m model\u001b[38;5;241m.\u001b[39mload_state_dict(copy\u001b[38;5;241m.\u001b[39mdeepcopy(model\u001b[38;5;241m.\u001b[39mstate_dict()))\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# save quantized model using safetensors\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave_quantized\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquantized_model_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muse_safetensors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/aigc-nas02/workspace/online/llm_infer/GPTQModel/gptqmodel/models/base.py:625\u001b[0m, in \u001b[0;36msave_quantized\u001b[0;34m(self, save_dir, safetensors_metadata, use_safetensors, max_shard_size, model_base_name)\u001b[0m\n\u001b[1;32m    621\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m quantize_config\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;241m==\u001b[39m FORMAT\u001b[38;5;241m.\u001b[39mGPTQ:\n\u001b[1;32m    622\u001b[0m     \u001b[38;5;66;03m# Model qzeros may be edited in place.\u001b[39;00m\n\u001b[1;32m    623\u001b[0m     \u001b[38;5;66;03m# TODO: avoid inplace modification of the weights\u001b[39;00m\n\u001b[1;32m    624\u001b[0m     model \u001b[38;5;241m=\u001b[39m copy\u001b[38;5;241m.\u001b[39mdeepcopy(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel)\n\u001b[0;32m--> 625\u001b[0m     model \u001b[38;5;241m=\u001b[39m convert_gptq_v2_to_v1_format(\n\u001b[1;32m    626\u001b[0m         model, quantize_config\u001b[38;5;241m=\u001b[39mquantize_config, qlinear_kernel\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqlinear_kernel\n\u001b[1;32m    627\u001b[0m     )\n\u001b[1;32m    629\u001b[0m model\u001b[38;5;241m.\u001b[39mto(CPU)\n\u001b[1;32m    631\u001b[0m state_dict \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mstate_dict()\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (1 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (1 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (7 times), _deepcopy_dict at line 230 (3 times), _reconstruct at line 270 (3 times), _reconstruct at line 296 (3 times), deepcopy at line 146 (3 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:153\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    151\u001b[0m copier \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(x, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__deepcopy__\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    152\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 153\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    154\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    155\u001b[0m     reductor \u001b[38;5;241m=\u001b[39m dispatch_table\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/site-packages/torch/_tensor.py:86\u001b[0m, in \u001b[0;36mTensor.__deepcopy__\u001b[0;34m(self, memo)\u001b[0m\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(Tensor\u001b[38;5;241m.\u001b[39m__deepcopy__, (\u001b[38;5;28mself\u001b[39m,), \u001b[38;5;28mself\u001b[39m, memo)\n\u001b[1;32m     85\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_leaf:\n\u001b[0;32m---> 86\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m     87\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOnly Tensors created explicitly by the user \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     88\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m(graph leaves) support the deepcopy protocol at the moment.  \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     89\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIf you were attempting to deepcopy a module, this may be because \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     90\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mof a torch.nn.utils.weight_norm usage, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     91\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msee https://github.com/pytorch/pytorch/pull/103001\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     92\u001b[0m     )\n\u001b[1;32m     93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mid\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;129;01min\u001b[39;00m memo:\n\u001b[1;32m     94\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m memo[\u001b[38;5;28mid\u001b[39m(\u001b[38;5;28mself\u001b[39m)]\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment.  If you were attempting to deepcopy a module, this may be because of a torch.nn.utils.weight_norm usage, see https://github.com/pytorch/pytorch/pull/103001"
+     ]
+    }
+   ],
+   "source": [
+    "from torch import device\n",
+    "model.to(device(\"cpu\"))\n",
+    "import copy\n",
+    "model.load_state_dict(copy.deepcopy(model.state_dict()))\n",
+    "\n",
+    "# save quantized model using safetensors\n",
+    "model.save_quantized(quantized_model_dir, use_safetensors=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "7eda7b80-a12e-46ed-a98f-21737a7fac15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def shallow_copy(x: dict):\n",
+    "    c = {}\n",
+    "    for k,v in x.items():\n",
+    "        c[k] = clone_if_tensor(v)\n",
+    "    return c\n",
+    "\n",
+    "\n",
+    "def clone_if_tensor(value):\n",
+    "    import torch\n",
+    "    if isinstance(value, torch.Tensor):\n",
+    "        return value.clone()\n",
+    "    elif isinstance(value, list):\n",
+    "        return [clone_if_tensor(v) for v in value]\n",
+    "    elif isinstance(value, dict):\n",
+    "        return {k: clone_if_tensor(v) for k, v in value.items()}\n",
+    "    elif isinstance(value, (int, float, bool, str)):\n",
+    "        return copy.deepcopy(value)\n",
+    "    else:\n",
+    "        return value"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "57ee1356-9b21-4966-8830-66aa9621490e",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment.  If you were attempting to deepcopy a module, this may be because of a torch.nn.utils.weight_norm usage, see https://github.com/pytorch/pytorch/pull/103001",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[30], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m model\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mload_state_dict(shallow_copy(model\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mstate_dict()))\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m# save quantized model using safetensors\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave_quantized\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquantized_model_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muse_safetensors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/aigc-nas02/workspace/online/llm_infer/GPTQModel/gptqmodel/models/base.py:625\u001b[0m, in \u001b[0;36msave_quantized\u001b[0;34m(self, save_dir, safetensors_metadata, use_safetensors, max_shard_size, model_base_name)\u001b[0m\n\u001b[1;32m    621\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m quantize_config\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;241m==\u001b[39m FORMAT\u001b[38;5;241m.\u001b[39mGPTQ:\n\u001b[1;32m    622\u001b[0m     \u001b[38;5;66;03m# Model qzeros may be edited in place.\u001b[39;00m\n\u001b[1;32m    623\u001b[0m     \u001b[38;5;66;03m# TODO: avoid inplace modification of the weights\u001b[39;00m\n\u001b[1;32m    624\u001b[0m     model \u001b[38;5;241m=\u001b[39m copy\u001b[38;5;241m.\u001b[39mdeepcopy(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel)\n\u001b[0;32m--> 625\u001b[0m     model \u001b[38;5;241m=\u001b[39m convert_gptq_v2_to_v1_format(\n\u001b[1;32m    626\u001b[0m         model, quantize_config\u001b[38;5;241m=\u001b[39mquantize_config, qlinear_kernel\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqlinear_kernel\n\u001b[1;32m    627\u001b[0m     )\n\u001b[1;32m    629\u001b[0m model\u001b[38;5;241m.\u001b[39mto(CPU)\n\u001b[1;32m    631\u001b[0m state_dict \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mstate_dict()\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (1 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (1 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "    \u001b[0;31m[... skipping similar frames: deepcopy at line 172 (7 times), _deepcopy_dict at line 230 (3 times), _reconstruct at line 270 (3 times), _reconstruct at line 296 (3 times), deepcopy at line 146 (3 times)]\u001b[0m\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:270\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m state \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m deep:\n\u001b[0;32m--> 270\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(y, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__setstate__\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    272\u001b[0m         y\u001b[38;5;241m.\u001b[39m__setstate__(state)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:146\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    144\u001b[0m copier \u001b[38;5;241m=\u001b[39m _deepcopy_dispatch\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 146\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;28mtype\u001b[39m):\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:230\u001b[0m, in \u001b[0;36m_deepcopy_dict\u001b[0;34m(x, memo, deepcopy)\u001b[0m\n\u001b[1;32m    228\u001b[0m memo[\u001b[38;5;28mid\u001b[39m(x)] \u001b[38;5;241m=\u001b[39m y\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 230\u001b[0m     y[deepcopy(key, memo)] \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:172\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    170\u001b[0m                 y \u001b[38;5;241m=\u001b[39m x\n\u001b[1;32m    171\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 172\u001b[0m                 y \u001b[38;5;241m=\u001b[39m \u001b[43m_reconstruct\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mrv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m# If is its own copy, don't memoize.\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m y \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m x:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:296\u001b[0m, in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dictiter:\n\u001b[1;32m    295\u001b[0m         key \u001b[38;5;241m=\u001b[39m deepcopy(key, memo)\n\u001b[0;32m--> 296\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mdeepcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    297\u001b[0m         y[key] \u001b[38;5;241m=\u001b[39m value\n\u001b[1;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/copy.py:153\u001b[0m, in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    151\u001b[0m copier \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(x, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__deepcopy__\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    152\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m copier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 153\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[43mcopier\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmemo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    154\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    155\u001b[0m     reductor \u001b[38;5;241m=\u001b[39m dispatch_table\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mcls\u001b[39m)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/site-packages/torch/_tensor.py:86\u001b[0m, in \u001b[0;36mTensor.__deepcopy__\u001b[0;34m(self, memo)\u001b[0m\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(Tensor\u001b[38;5;241m.\u001b[39m__deepcopy__, (\u001b[38;5;28mself\u001b[39m,), \u001b[38;5;28mself\u001b[39m, memo)\n\u001b[1;32m     85\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_leaf:\n\u001b[0;32m---> 86\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m     87\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOnly Tensors created explicitly by the user \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     88\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m(graph leaves) support the deepcopy protocol at the moment.  \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     89\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIf you were attempting to deepcopy a module, this may be because \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     90\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mof a torch.nn.utils.weight_norm usage, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     91\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msee https://github.com/pytorch/pytorch/pull/103001\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     92\u001b[0m     )\n\u001b[1;32m     93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mid\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;129;01min\u001b[39;00m memo:\n\u001b[1;32m     94\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m memo[\u001b[38;5;28mid\u001b[39m(\u001b[38;5;28mself\u001b[39m)]\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment.  If you were attempting to deepcopy a module, this may be because of a torch.nn.utils.weight_norm usage, see https://github.com/pytorch/pytorch/pull/103001"
+     ]
+    }
+   ],
+   "source": [
+    "model.model.load_state_dict(shallow_copy(model.model.state_dict()))\n",
+    "# save quantized model using safetensors\n",
+    "model.save_quantized(quantized_model_dir, use_safetensors=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "ad3bb208-f1b6-4eee-a2e5-2db7cc046751",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "'Qwen2ForCausalLM' object has no attribute 'weight'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[29], line 6\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# model = utils.weight_norm(model.state_dict())\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m model2 \u001b[38;5;241m=\u001b[39m \u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight_norm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;66;03m# 在进行深拷贝前移除权重规范化\u001b[39;00m\n\u001b[1;32m      9\u001b[0m utils\u001b[38;5;241m.\u001b[39mremove_weight_norm(model2)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/site-packages/torch/nn/utils/weight_norm.py:130\u001b[0m, in \u001b[0;36mweight_norm\u001b[0;34m(module, name, dim)\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mweight_norm\u001b[39m(module: T_module, name: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mweight\u001b[39m\u001b[38;5;124m'\u001b[39m, dim: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T_module:\n\u001b[1;32m     71\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Apply weight normalization to a parameter in the given module.\u001b[39;00m\n\u001b[1;32m     72\u001b[0m \n\u001b[1;32m     73\u001b[0m \u001b[38;5;124;03m    .. math::\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    128\u001b[0m \n\u001b[1;32m    129\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m     \u001b[43mWeightNorm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodule\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    131\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m module\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/site-packages/torch/nn/utils/weight_norm.py:39\u001b[0m, in \u001b[0;36mWeightNorm.apply\u001b[0;34m(module, name, dim)\u001b[0m\n\u001b[1;32m     35\u001b[0m     dim \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m\n\u001b[1;32m     37\u001b[0m fn \u001b[38;5;241m=\u001b[39m WeightNorm(name, dim)\n\u001b[0;32m---> 39\u001b[0m weight \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mmodule\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(weight, UninitializedParameter):\n\u001b[1;32m     41\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m     42\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThe module passed to `WeightNorm` can\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124mt have uninitialized parameters. \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m     43\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMake sure to run the dummy forward before applying weight normalization\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "File \u001b[0;32m/home/work/miniconda3/envs/autoawq2/lib/python3.9/site-packages/torch/nn/modules/module.py:1709\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1707\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m   1708\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1709\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'Qwen2ForCausalLM' object has no attribute 'weight'"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.utils as utils\n",
+    "\n",
+    "# model = utils.weight_norm(model.state_dict())\n",
+    "model2 = utils.weight_norm(model.model)\n",
+    "\n",
+    "# 在进行深拷贝前移除权重规范化\n",
+    "utils.remove_weight_norm(model2)\n",
+    "\n",
+    "# 进行深拷贝\n",
+    "model_copy = copy.deepcopy(model2)\n",
+    "\n",
+    "# 重新应用权重规范化\n",
+    "model_copy = utils.weight_norm(model_copy)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "1da8dcdf-176f-4114-89e8-9a8d16508dd4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}