Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Model] Initialize Phi-3-vision support #4986

Merged
merged 38 commits into from
Jun 18, 2024
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
251752c
init phi3v support
Isotr0py May 22, 2024
70e7017
make phi3v work
Isotr0py May 23, 2024
618a2cb
remove debug code
Isotr0py May 23, 2024
ffb32fb
remove dropout from Phi3ImageEmbedding
Isotr0py May 24, 2024
76e6f8e
clean code
Isotr0py May 24, 2024
58330ba
optimize code structure
Isotr0py May 24, 2024
b61c2be
Add Phi3VImagePixelInputs
Isotr0py May 24, 2024
1e3e18c
refactor image embedding
Isotr0py May 26, 2024
591a9b8
format phi3v_example
Isotr0py May 26, 2024
b8599fc
Merge branch 'vllm-project:main' into phi3v
Isotr0py Jun 6, 2024
2e5dc27
refactor phi3v
Isotr0py Jun 8, 2024
e7fe213
Merge branch 'vllm-project:main' into phi3v
Isotr0py Jun 8, 2024
1b1f3f2
remove phi3v feature inputs
Isotr0py Jun 8, 2024
1a9e43f
Merge branch 'vllm-project:main' into phi3v
Isotr0py Jun 11, 2024
8db0d25
refactor phi3v
Isotr0py Jun 11, 2024
7388bcd
deprecate phi3v image_input
Isotr0py Jun 12, 2024
3f3d2b8
add phi3v test
Isotr0py Jun 12, 2024
0705a62
fix phi3v test
Isotr0py Jun 13, 2024
5f21d3f
format code
Isotr0py Jun 13, 2024
fe4e594
Merge branch 'vllm-project:main' into phi3v
Isotr0py Jun 13, 2024
3b7f86a
add phi3_v to get_full_image_text_prompt and test marker
Isotr0py Jun 13, 2024
ced2c3d
add docs
Isotr0py Jun 13, 2024
1d82bb1
clear phi3v model implement
Isotr0py Jun 14, 2024
4739c45
ignore phi3v cpu test
Isotr0py Jun 14, 2024
59fe2c1
fix doc strings
Isotr0py Jun 14, 2024
38ed4d9
fix phi3v test flash_attn import
Isotr0py Jun 14, 2024
d2fbecf
fix phi3v test
Isotr0py Jun 14, 2024
d99b684
Merge branch 'main' into phi3v
Isotr0py Jun 14, 2024
2bbaecd
add torchvision to requirements-test.txt
Isotr0py Jun 14, 2024
ce62fad
increase phi3v max_model_len to 2048
Isotr0py Jun 15, 2024
4b242dd
Merge branch 'vllm-project:main' into phi3v
Isotr0py Jun 15, 2024
1d78590
decrease phi3v max_tokens to 8
Isotr0py Jun 15, 2024
7a3ba90
Merge branch 'vllm-project:main' into phi3v
Isotr0py Jun 15, 2024
b95672f
Merge branch 'vllm-project:main' into phi3v
Isotr0py Jun 17, 2024
da1392c
optimize image embedding and update requirements.txt
Isotr0py Jun 17, 2024
9c080be
remove changing input_ids to -1
Isotr0py Jun 17, 2024
0cd9d26
fix a typo and image embedding
Isotr0py Jun 17, 2024
e77bb76
update comment for phi3v
Isotr0py Jun 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions examples/phi3v_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import os
import subprocess

from PIL import Image
from transformers import AutoProcessor

from vllm import LLM, SamplingParams
from vllm.multimodal.image import ImagePixelData


def run_phi3v():
model_path = "microsoft/Phi-3-vision-128k-instruct"
processor = AutoProcessor.from_pretrained(model_path,
trust_remote_code=True)
llm = LLM(
model=model_path,
trust_remote_code=True,
image_input_type="pixel_values",
image_token_id=-1,
image_input_shape="1,3,1008,1344",
image_feature_size=1024,
disable_image_processor=False,
)

image = Image.open("images/stop_sign.jpg")
user_prompt = "<|user|>\n"
assistant_prompt = "<|assistant|>\n"
suffix = "<|end|>\n"

# single-image prompt
prompt = "What is shown in this image?"
prompt = f"{user_prompt}<|image_1|>\n{prompt}{suffix}{assistant_prompt}"

inputs = processor(prompt, image, return_tensors="pt")
Isotr0py marked this conversation as resolved.
Show resolved Hide resolved
input_ids = inputs["input_ids"].squeeze(0).tolist()
sampling_params = SamplingParams(temperature=0, max_tokens=64)

outputs = llm.generate({
"prompt_token_ids": input_ids,
"sampling_params": sampling_params,
"multi_modal_data": ImagePixelData(image),
})
for o in outputs:
generated_text = o.outputs[0].text
print(generated_text)


if __name__ == "__main__":
s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
local_directory = "images"

# Make sure the local directory exists or create it
os.makedirs(local_directory, exist_ok=True)

# Use AWS CLI to sync the directory, assume anonymous access
subprocess.check_call([
"aws",
"s3",
"sync",
s3_bucket_path,
local_directory,
"--no-sign-request",
])
run_phi3v()
1 change: 1 addition & 0 deletions vllm/model_executor/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
"Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
Expand Down
Loading
Loading