Merge remote-tracking branch 'gitee/lyh-merge2' into lyh-merge2

configs/tests/test_combinations.py

@@ -0,0 +1,68 @@
+from mmengine.config import read_base
+from opencompass.partitioners import SizePartitioner, NaivePartitioner
+from opencompass.runners import LocalRunner
+from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
+
+with read_base():
+    # Datasets
+    from ..datasets.opseval.datasets import zte_mc, oracle_mc, owl_mc, owl_qa, network_mc, company_mc, rzy_qa
+    # Models
+    from ..models.gpt_3dot5_turbo_peiqi import models as chatgpt
+    from ..local_models.zhipu.chatglm import chatglm3_6b
+    from ..local_models.internlm.internlm import internlm2_chat_20b, internlm2_chat_7b
+    from ..local_models.google.gemma import gemma_2b, gemma_7b
+    from ..local_models.qwen.qwen import qwen1_5_0_5b_base, qwen1_5_0_5b_chat
+    from ..local_models.baichuan.baichuan import baichuan2_turbo, baichuan3
+    from ..local_models.zhipu.zhipu import glm_3_turbo, glm_4
+    from ..paths import ROOT_DIR
+
+datasets = [
+    *zte_mc
+]
+
+models = [
+    qwen1_5_0_5b_base
+]
+
+for model in models:
+    model['run_cfg'] = dict(num_gpus=1, num_procs=1)
+    pass
+
+for dataset in datasets:
+    dataset['sample_setting'] = dict()
+    dataset['infer_cfg']['inferencer']['save_every'] = 8
+    dataset['infer_cfg']['inferencer']['sc_size'] = 2
+    dataset['eval_cfg']['sc_size'] = 2
+    dataset['sample_setting'] = dict(sample_size=2)     # !!!WARNING: Use for testing only!!!
+
+
+model_dataset_combinations = [
+    dict(models=[model for model in models if 'chat' not in model['abbr']], 
+         datasets=[dataset for dataset in datasets if 'ppl' in dataset['abbr']]),
+    dict(models=[model for model in models if 'chat' in model['abbr']],
+         datasets=[dataset for dataset in datasets if 'gen' in dataset['abbr']])
+]
+
+
+infer = dict(
+    partitioner=dict(
+        # type=SizePartitioner,
+        # max_task_size=100,
+        # gen_task_coef=1,
+        type=NaivePartitioner
+    ),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=16,
+        max_workers_per_gpu=1,
+        task=dict(type=OpenICLInferTask),
+    ),
+)
+
+eval = dict(
+    partitioner=dict(type=NaivePartitioner),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=32,
+        task=dict(type=OpenICLEvalTask)),
+)

configs/xz/generate.py

@@ -42,6 +42,12 @@
         ],
     )"""
 }
+meta_template = dict(
+        round=[
+            dict(role="HUMAN", begin='<|im_start|>user\\n', end='<|im_end|>'),
+            dict(role="BOT", begin="<|im_start|>assistant\\n", end='<|im_end|>', generate=True),
+        ],
+    )
 
 eos_tokens_dict = {
     "qwen": """[151643, 151645]""",

configs/xz/runconfig.py

@@ -13,9 +13,9 @@
     from ..datasets.ppl_qa.rzy_qa import rzy_ppl_qa_datasets
     from ..datasets.zte.zte import zte_naive
 datasets = [*ceval_mc_ppl,*network_mc_ppl,*zte_mc_ppl,*owl_mc_ppl,*oracle_mc_ppl,*company_mc_ppl,*ceval_mc_gen,*network_mc_gen,*zte_mc_gen,*owl_mc_gen,*oracle_mc_gen,*company_mc_gen,*owl_qa_gen,*owl_qa_ppl,*rzy_qa_gen,*rzy_qa_ppl]
-model_name = 'qwen-14b-chat'
-model_abbr = 'nm_qwen_14b_chat'
-model_path = '/mnt/home/opsfm-xz/models/Qwen/Qwen-14B-Chat'
+model_name = 'yi-34b-dsir_150000-full-owl-network-sft-2000steps'
+model_abbr = 'nm_yi_34b_dsir_150000_full_owl_network_sft_2000steps'
+model_path = '/mnt/tenant-home_speed/xz/sft_checkpoint/yi-34b-dsir_150000-full-owl-network-sft-2000steps/merged_model'
 
 if model_name is None:
     raise NotImplementedError("Model is none!")
@@ -26,7 +26,7 @@
             engine_config=dict(session_len=2048,
                            max_batch_size=8),
             gen_config=dict(top_k=1, top_p=0.8,
-                        max_new_tokens=100, stop_words=[151643, 151645]),
+                        max_new_tokens=100, stop_words=[2, 7]),
             max_out_len=400,
             max_seq_len=2048,
             batch_size=8,
@@ -36,7 +36,6 @@
             dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>'),
             dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>', generate=True),
         ],
-        reserved_roles=[dict(role='SYSTEM', begin="<|im_start|>system\nYou are a helpful assistant.", end="<|im_end|>"),]
     ),
             end_str='<|im_end|>'
         )]

opencompass/openicl/icl_evaluator/opseval_gen_evaluator.py

@@ -119,7 +119,7 @@ def score(self, predictions: List, references: List) -> dict:
             'Accuracy': correct / tot * 100,
             'SC-Accuracy': sc_correct / tot * 100,
         }
-    
+
 class OpsEvalGenQAEvaluator(BaseEvaluator):
 
     def __init__(self, language='en'):
@@ -155,4 +155,4 @@ def clean_word(words):
         except Exception as err:
             print(f"[WARNING] Error when calculating bleu and rouge: {err}")
             bleu_score, rouge_score = 0.0, 0.0
-        return bleu_score, rouge_score
+        return bleu_score, rouge_score