add necessary comments

Luodian · Sep 11, 2023 · 4bd8064 · 4bd8064
1 parent ac0cfcd
commit 4bd8064
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/pipeline/evaluation/datasets/mmbench.py b/pipeline/evaluation/datasets/mmbench.py
@@ -17,7 +17,6 @@ def decode_base64_to_image(base64_string):
 class MMBenchDataset(Dataset):
     def __init__(self, data_file, sys_prompt="There are several options:"):
         self.df = pd.read_csv(data_file, sep="\t")
-        # self.pipeline = Compose(pipeline)
         self.sys_prompt = sys_prompt
 
     def __len__(self):

diff --git a/pipeline/evaluation/evaluate.py b/pipeline/evaluation/evaluate.py
@@ -4,7 +4,6 @@
 
 
 from pipeline.evaluation.evaluator.mmbench import MMBenchEvaluator
-
 from pipeline.evaluation.models.idefics import Idefics
 
 
@@ -13,3 +12,7 @@
     evaluator = MMBenchEvaluator("/data/pufanyi/training_data/MMBench/mmbench_test_20230712.tsv")
     model = Idefics("/data/pufanyi/training_data/checkpoints/idefics-9b-instruct")
     evaluator.evaluate(model)
+
+# pip install otter_ai
+# other necessary packages
+# python -m otter_ai.eval --models=Otter --model_path=luodian/OTTER-Image-MPT --dataset=MMBench
diff --git a/pipeline/evaluation/evaluator/mmbench.py b/pipeline/evaluation/evaluator/mmbench.py
@@ -57,6 +57,10 @@ def __init__(self, data_file, sys_prompt="There are several options:"):
             )
 
     def evaluate(self, model, output_file=None):
+        # inside this function
+        # MMBench: generate a csv file with columns: index, prediction, xxx
+        # MME: generate the final score (probably should load the gt file) and print it out
+        # SeedBench: generate the final score (probably should load the gt file) and print it out
         if output_file is None:
             output_file = self.default_output_file