add otter video

Luodian · Sep 13, 2023 · 5541381 · 5541381
1 parent 883057d
commit 5541381
Show file tree

Hide file tree

Showing 8 changed files with 4 additions and 238 deletions.
diff --git a/pipeline/evaluation/evaluate.py b/pipeline/evaluation/evaluate.py
@@ -8,7 +8,7 @@
 if __name__ == "__main__":
     evaluator = load_dataset("mhbench", {"dataset_path": "ZhangYuanhan/multi-hop-reasoning"})
     # model = load_model("llama_adapter", {"model_path": "/mnt/petrelfs/zhangyuanhan/Otter/pipeline/evaluation/ckpts/7B.pth"})
-    model = load_model("video_chat", {"model_path": ""})
+    model = load_model("otter_video", {"model_path": "/mnt/petrelfs/zhangyuanhan/Otter/checkpoint/OTTER-9B-DenseCaption"})
     evaluator.evaluate(model)
 
 # pip install otter_ai

diff --git a/pipeline/evaluation/models/base_model.py b/pipeline/evaluation/models/base_model.py
@@ -4,7 +4,7 @@
 
 import importlib
 
-AVAILABLE_MODELS: Dict[str, str] = {"video_chat": "VideoChat", "llama_adapter": "LlamaAdapter"}
+AVAILABLE_MODELS: Dict[str, str] = {"video_chat": "VideoChat", "llama_adapter": "LlamaAdapter", "otter_video": "OtterVideo"}
 
 
 class BaseModel(ABC):

diff --git a/pipeline/evaluation/models/idefics.py b/pipeline/evaluation/models/idefics.py
diff --git a/pipeline/evaluation/models/idefics_otter.py b/pipeline/evaluation/models/idefics_otter.py
diff --git a/pipeline/evaluation/models/otter.py b/pipeline/evaluation/models/otter.py
diff --git a/pipeline/evaluation/models/otter_video.py b/pipeline/evaluation/models/otter_video.py
diff --git a/pipeline/evaluation/models/run_idefics.py b/pipeline/evaluation/models/run_idefics.py
diff --git a/src/otter_ai/models/otter/configuration_otter.py b/src/otter_ai/models/otter/configuration_otter.py
@@ -5,6 +5,8 @@
 from transformers.models.auto import CONFIG_MAPPING
 from transformers.models.clip import CLIPVisionConfig
 
+import sys
+sys.path.append("/mnt/petrelfs/zhangyuanhan/Otter/src")
 from otter_ai.models.flamingo.falcon.configuration_RW import RWConfig
 from otter_ai.models.flamingo.mpt.configuration_mpt import MPTConfig
 from otter_ai.models.flamingo.mpt_redpajama.configuration_mosaic_gpt import MosaicGPTConfig