Add microsoft/phi-1_5 qlora example

examples/open_llama/README.md

@@ -83,7 +83,7 @@ The relevant config file is [llama_qlora.json](llama_qlora.json). It corresponds
 Requirements file: [requirements-lora.txt](requirements-lora.txt)
 
 ### Fine-tune Open Llama Model on a code generation dataset using LoRA
-This workflow fine-tunes Open LLaMA model using [LoRA] to generate code given a prompt.
+This workflow fine-tunes Open LLaMA model using [LoRA](https://arxiv.org/abs/2106.09685) to generate code given a prompt.
 
 The relevant config file is [open_llama_lora_tinycodes.json](open_llama_lora_tinycodes.json). The code language is set to `Python` but can be changed to other languages by changing the `language` field in the config file.
 Supported languages are Python, TypeScript, JavaScript, Ruby, Julia, Rust, C++, Bash, Java, C#, and Go. Refer to the [dataset card](https://huggingface.co/datasets/nampdn-ai/tiny-codes) for more details on the dataset.

examples/open_llama/llama_qlora.json

@@ -71,6 +71,6 @@
         },
         "execution_providers": ["CPUExecutionProvider"],
         "cache_dir": "cache",
-        "output_dir" : "models/qlora"
+        "output_dir" : "models/huggyllama_qlora"
     }
 }

examples/open_llama/open_llama_inc_woq.json

@@ -79,6 +79,6 @@
         "evaluator": "common_evaluator",
         "cache_dir": "cache",
         "output_name": "ollama",
-        "output_dir": "footprints"
+        "output_dir": "models/open_llama_inc_woq"
     }
 }

examples/open_llama/open_llama_lora_tinycodes.json

@@ -69,6 +69,6 @@
         },
         "execution_providers": ["CPUExecutionProvider"],
         "cache_dir": "cache",
-        "output_dir" : "models/llama/lora-tiny-codes"
+        "output_dir" : "models/open_llama_lora_tinycodes"
     }
 }

examples/open_llama/open_llama_qlora_tinycodes.json

@@ -69,6 +69,6 @@
         },
         "execution_providers": ["CPUExecutionProvider"],
         "cache_dir": "cache",
-        "output_dir" : "models/qlora-tiny-codes"
+        "output_dir" : "models/open_llama_qlora_tinycodes"
     }
 }

examples/open_llama/open_llama_sparsegpt_gpu.json

@@ -90,6 +90,6 @@
         },
         "execution_providers": ["CPUExecutionProvider"],
         "cache_dir": "cache",
-        "output_dir" : "models/sparsegpt"
+        "output_dir" : "models/open_llama_sparsegpt"
     }
 }

examples/phi/README.md

@@ -0,0 +1,42 @@
+# phi-1.5 Optimization
+This folder contains examples of phi-1.5 workflow.
+
+## Optimization Workflows
+### Fine-tune  phi-1.5 Model on a code generation dataset using QLoRA
+This workflow fine-tunes [phi-1.5 model](https://huggingface.co/microsoft/phi-1_5) using [QLoRA](https://arxiv.org/abs/2305.14314) to generate code given a prompt.
+
+The relevant config file is [phi_qlora_tinycodes.json](phi_qlora_tinycodes.json). The code language is set to `Python` but can be changed to other languages by changing the `language` field in the config file.
+Supported languages are Python, TypeScript, JavaScript, Ruby, Julia, Rust, C++, Bash, Java, C#, and Go. Refer to the [dataset card](https://huggingface.co/datasets/nampdn-ai/tiny-codes) for more details on the dataset.
+
+Note: You must be logged in to HuggingFace using `huggingface-cli login` to download the dataset or update `token` field in the config file with your HuggingFace token.
+
+Requirements file: [requirements-lora.txt](requirements-lora.txt)
+
+## How to run
+### Pip requirements
+Install the necessary python packages using the corresponding requirements file.
+```
+python -m pip install -r <requirements_file>.txt
+```
+
+### Run sample using config
+
+The optimization techniques to run are specified in the relevant config json file.
+
+First, install required packages according to passes.
+```
+python -m olive.workflows.run --config <config_file>.json --setup
+```
+
+Then, optimize the model
+```
+python -m olive.workflows.run --config <config_file>.json
+```
+
+or run simply with python code:
+```python
+from olive.workflows import run as olive_run
+olive_run("<config_file>.json")
+```
+
+After running the above command, the model candidates and corresponding config will be saved in the output directory.

examples/phi/phi_qlora_tinycodes.json

@@ -0,0 +1,78 @@
+{
+    "input_model":{
+        "type": "PyTorchModel",
+        "config": {
+            "hf_config": {
+                "model_name": "microsoft/phi-1_5",
+                "task": "text-generation",
+                "model_loading_args": {
+                    "trust_remote_code": true
+                }
+            }
+        }
+    },
+    "data_configs": {
+        "tiny-codes-train": {
+            "name": "tiny-codes-train",
+            "type": "HuggingfaceContainer",
+            "user_script": "user_script.py",
+            "components": {
+                "load_dataset": {
+                    "type": "load_tiny_code_dataset"
+                }
+            },
+            "params_config": {
+                "data_name": "nampdn-ai/tiny-codes",
+                "split": "train",
+                "component_kwargs": {
+                    "load_dataset": {
+                        "language": "Python",
+                        "token": true
+                    },
+                    "pre_process_data": {
+                        "dataset_type": "corpus",
+                        "corpus_strategy": "join",
+                        "text_template": "### Question: {prompt} \n### Answer: {response}",
+                        "source_max_len": 1024,
+                        "use_attention_mask": false
+                    }
+                }
+            }
+        }
+    },
+    "passes": {
+        "qlora": {
+            "type": "QLoRA",
+            "config": {
+                "lora_dropout": 0.1,
+                "train_data_config": "tiny-codes-train",
+                "eval_dataset_size": 1024,
+                "training_args": {
+                    "per_device_train_batch_size": 4,
+                    "per_device_eval_batch_size": 4,
+                    "gradient_accumulation_steps": 4,
+                    "max_steps": 1500,
+                    "logging_steps": 100,
+                    "evaluation_strategy": "steps",
+                    "adam_beta2": 0.999,
+                    "max_grad_norm": 0.3,
+                    "load_best_model_at_end": true
+                }
+            }
+        }
+    },
+    "engine": {
+        "log_severity_level": 0,
+        "search_strategy": false,
+        "evaluate_input_model": false,
+        "target": {
+            "type": "LocalSystem",
+            "config": {
+                "accelerators": ["gpu"]
+            }
+        },
+        "execution_providers": ["CPUExecutionProvider"],
+        "cache_dir": "cache",
+        "output_dir" : "models/phi_qlora_tinycodes"
+    }
+}

examples/phi/requirements-lora.txt

@@ -0,0 +1,5 @@
+-r requirements.txt
+accelerate
+bitsandbytes
+peft
+scikit-learn

examples/phi/requirements.txt

@@ -0,0 +1,3 @@
+datasets
+einops
+sentencepiece

examples/phi/user_script.py

@@ -0,0 +1,16 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+from typing import Union
+
+from datasets import load_dataset
+
+from olive.data.registry import Registry
+
+
+# TODO(jambayk): remove custom dataset component once default dataset component supports filter, tokens and split
+@Registry.register_dataset()
+def load_tiny_code_dataset(data_name: str, split: str, language: str, token: Union[bool, str] = True):
+    dataset = load_dataset(data_name, split=split, token=token)
+    return dataset.filter(lambda x: x["programming_language"] == language)