diff --git a/paddlenlp/trainer/trainer.py b/paddlenlp/trainer/trainer.py index 2f3aad5e712e..416786721462 100644 --- a/paddlenlp/trainer/trainer.py +++ b/paddlenlp/trainer/trainer.py @@ -1795,17 +1795,8 @@ def _wrap_model(self, model, training=True): in_cp_parallel_mode = self.args.context_parallel_degree > 1 # Multi-gpu training - if ( - self.args.world_size > 1 - and not self.args.use_hybrid_parallel - or not ( - in_pipeline_parallel_mode - or in_sharding_parallel_mode - or in_tensor_parallel_mode - or in_sep_parallel_mode - or in_cp_parallel_mode - ) - ): + if self.args.world_size > 1 and (not self.args.use_hybrid_parallel): + # MOE use DDP to broadcaset parameters. model = paddle.DataParallel(model) # Distributed training (should be after fp16 initialization) diff --git a/paddlenlp/trainer/training_args.py b/paddlenlp/trainer/training_args.py index 0ff0c4b60319..cdb9659f2ce4 100644 --- a/paddlenlp/trainer/training_args.py +++ b/paddlenlp/trainer/training_args.py @@ -1529,7 +1529,7 @@ def is_segment_parallel_supported(): if world_size > 1: if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized(): if self.unified_checkpoint: - self.use_hybrid_parallel = True + # DP use hybrid group strategy = fleet.DistributedStrategy() fleet.init(is_collective=True, strategy=strategy) else: diff --git a/pyproject.toml b/pyproject.toml index eb8cd6e438ff..b3cbf1d383d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,6 @@ exclude = ['.flake8'] [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra -q --dist loadgroup" -retries = 0 -retry_delay = 0.5 -timeout = 200 pythonpath = ["."] testpaths = [ "tests/data", @@ -25,7 +22,7 @@ testpaths = [ "tests/layers", "tests/metrics", "tests/ops", - # "tests/trainer", + "tests/trainer", "tests/transformers", "tests/peft", "tests/prompt", diff --git a/scripts/unit_test/ci_unit.sh b/scripts/unit_test/ci_unit.sh index e707aeed4885..1bb37982b586 100644 --- a/scripts/unit_test/ci_unit.sh +++ b/scripts/unit_test/ci_unit.sh @@ -30,10 +30,10 @@ install_requirements() { python -m pip install -r requirements-dev.txt python -m pip install -r tests/requirements.txt python -m pip install -r paddlenlp/experimental/autonlp/requirements.txt - python -m pip uninstall paddlepaddle -y + python -m pip uninstall paddlepaddle paddlepaddle_gpu -y python -m pip install --no-cache-dir ${paddle} - python setup.py bdist_wheel + python setup.py bdist_wheel > /dev/null python -m pip install dist/p****.whl cd csrc/ python setup_cuda.py install @@ -51,4 +51,4 @@ set_env() { install_requirements set_env -pytest -v -n 8 --durations 20 +pytest -v -n 8 --timeout 200 --durations 20 --cov paddlenlp --cov-report xml:coverage.xml diff --git a/tests/trainer/test_lora_unified_checkpoint.py b/tests/trainer/test_lora_unified_checkpoint.py index f22825dd09d2..3a5533f65c1f 100644 --- a/tests/trainer/test_lora_unified_checkpoint.py +++ b/tests/trainer/test_lora_unified_checkpoint.py @@ -151,7 +151,7 @@ def __test__(cls): def setUp(self): """ - 1. update runfrist and rerun to run defined different config + 1. update runfirst and rerun to run defined different config 2. update need_allclose to True if you want to check the result 3. update rtol to the relative value you want to check """ @@ -171,7 +171,7 @@ def setUp(self): self.run_lora_file = "llm/finetune_generation.py" - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_lora_file, **train_args) def rerun(self, train_args): @@ -183,7 +183,7 @@ def testTP4PP2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["TP4PP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -198,7 +198,7 @@ def testTP2Sharding4(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["TP2Sharding4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -216,7 +216,7 @@ def testTP8(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["TP8"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -230,7 +230,7 @@ def testTP4DP2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["TP4DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -245,7 +245,7 @@ def testTP4Sharding2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["TP4Sharding2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -260,7 +260,7 @@ def testTP2PP4(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["TP2PP4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -275,7 +275,7 @@ def testPP8(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["PP8"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -290,7 +290,7 @@ def testPP4DP2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["PP4DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -305,7 +305,7 @@ def testPP4Sharding2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["PP4Sharding2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -320,7 +320,7 @@ def testSharding8S1(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["Sharding8S1"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -335,7 +335,7 @@ def testSharding8S2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["Sharding8S2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -350,7 +350,7 @@ def testSharding4S1DP2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["Sharding4S1DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -365,7 +365,7 @@ def testSharding4S2DP2(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["Sharding4S2DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -380,7 +380,7 @@ def testSharding2S1DP4(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["Sharding2S1DP4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -395,7 +395,7 @@ def testSharding2S2DP4(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["Sharding2S2DP4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -410,7 +410,7 @@ def testDP8(self): remove_ckpt(lora_arguments["output_dir"]) train_args = self.configs["DP8"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -419,19 +419,21 @@ def testDP8(self): np.testing.assert_allclose(res[0], res[1], self.rtol) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestUnifiedCheckpointOnN2C4(TestUnifiedCheckpointBase): def setUp(self): super().setUp() self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_lora_file, **train_args) def rerun(self, train_args): self.run_n2c4(self.run_lora_file, **train_args) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestUnifiedCheckpointOnN1C8CheckpointCompatible(TestUnifiedCheckpointBase): def setUp(self): super().setUp() @@ -439,7 +441,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 0 self.run_n1c8(self.run_lora_file, **train_args) @@ -448,6 +450,7 @@ def rerun(self, train_args): self.run_n1c8(self.run_lora_file, **train_args) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestPaddleCheckpointOnN1C8Reset(TestUnifiedCheckpointBase): def setUp(self): super().setUp() @@ -455,7 +458,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 0 self.run_n1c8(self.run_lora_file, **train_args) @@ -472,7 +475,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 0 self.run_n2c4(self.run_lora_file, **train_args) diff --git a/tests/trainer/test_unified_checkpoint.py b/tests/trainer/test_unified_checkpoint.py index a5e4563d0317..640a1d70307c 100644 --- a/tests/trainer/test_unified_checkpoint.py +++ b/tests/trainer/test_unified_checkpoint.py @@ -177,7 +177,7 @@ def __test__(cls): def setUp(self): """ - 1. update runfrist and rerun to run defined diffrent config + 1. update runfirst and rerun to run defined diffrent config 2. update need_allclose to True if you want to check the result 3. update rtol to the relative value you want to check """ @@ -196,7 +196,7 @@ def setUp(self): self.run_pretrain_file = "llm/llama/run_pretrain.py" - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -208,7 +208,7 @@ def testTP4PP2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["TP4PP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -223,7 +223,7 @@ def testTP2Sharding4(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["TP2Sharding4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -241,7 +241,7 @@ def testTP8(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["TP8"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -255,7 +255,7 @@ def testTP4DP2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["TP4DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -270,7 +270,7 @@ def testTP4Sharding2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["TP4Sharding2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -285,7 +285,7 @@ def testTP2PP4(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["TP2PP4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -300,7 +300,7 @@ def testPP8(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["PP8"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -315,7 +315,7 @@ def testPP4DP2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["PP4DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -330,7 +330,7 @@ def testPP4Sharding2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["PP4Sharding2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -345,7 +345,7 @@ def testSharding8S1(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["Sharding8S1"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -360,7 +360,7 @@ def testSharding8S2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["Sharding8S2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -375,7 +375,7 @@ def testSharding4S1DP2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["Sharding4S1DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -390,7 +390,7 @@ def testSharding4S2DP2(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["Sharding4S2DP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -405,7 +405,7 @@ def testSharding2S1DP4(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["Sharding2S1DP4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -420,7 +420,7 @@ def testSharding2S2DP4(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["Sharding2S2DP4"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -435,7 +435,7 @@ def testDP8(self): remove_ckpt(pretrain_arguments["output_dir"]) train_args = self.configs["DP8"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -444,13 +444,14 @@ def testDP8(self): np.testing.assert_allclose(res[0], res[1], self.rtol) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestUnifiedCheckpointOnN2C4(TestUnifiedCheckpointBase): def setUp(self): super().setUp() self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -466,7 +467,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -488,7 +489,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -510,7 +511,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) move_checkpoint_N1C8_to_N2C4() @@ -532,7 +533,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_pretrain_file, **train_args) move_checkpoint_N2C4_to_N1C8() @@ -560,7 +561,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -579,7 +580,7 @@ def setUp(self): self.need_allclose = False - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O1" self.run_n1c8(self.run_pretrain_file, **train_args) @@ -588,6 +589,7 @@ def rerun(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestUnifiedCheckpointOnN1C8MasterWeightCompatibleO2ToO1(TestUnifiedCheckpointBase): def setUp(self): super().setUp() @@ -599,7 +601,7 @@ def setUp(self): self.need_allclose = False - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O2" self.run_n1c8(self.run_pretrain_file, **train_args) @@ -608,6 +610,7 @@ def rerun(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestUnifiedCheckpointOnN1C8CheckpointCompatible(TestUnifiedCheckpointBase): def setUp(self): super().setUp() @@ -615,7 +618,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 0 self.run_n1c8(self.run_pretrain_file, **train_args) @@ -624,6 +627,7 @@ def rerun(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestPaddleCheckpointOnN1C8Reset(TestUnifiedCheckpointBase): def setUp(self): super().setUp() @@ -631,7 +635,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 0 self.run_n1c8(self.run_pretrain_file, **train_args) @@ -640,6 +644,7 @@ def rerun(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestPaddleCheckpointOnN1C2Reset(TestMultipleGpus): def setUp(self): self.configs = get_pretrain_arguments(pretrain_arguments) @@ -656,7 +661,7 @@ def setUp(self): self.run_pretrain_file = "llm/llama/run_pretrain.py" - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 0 self.run_n1c2(self.run_pretrain_file, **train_args) @@ -672,7 +677,7 @@ def testTP2(self): train_args = self.configs["TP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -681,6 +686,7 @@ def testTP2(self): np.testing.assert_allclose(res[0], res[1], self.rtol) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestUnifiedCheckpointOnN1C2Reset(TestMultipleGpus): def setUp(self): self.configs = get_pretrain_arguments(pretrain_arguments) @@ -717,7 +723,7 @@ def setUp(self): "training_args.bin", ] - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 1 self.run_n1c2(self.run_pretrain_file, **train_args) @@ -733,7 +739,7 @@ def testTP2(self): train_args = self.configs["TP2"] - self.runfrist(train_args) + self.runfirst(train_args) self.rerun(train_args) if self.need_allclose: @@ -751,7 +757,7 @@ def testFileLists(self): base_ckpt_path = os.path.join(pretrain_arguments["output_dir"], "checkpoint-%d" % save_steps) train_args = self.configs["TP2"] - self.runfrist(train_args) + self.runfirst(train_args) assert sorted(self.filelists) == sorted(os.listdir(base_ckpt_path)) self.rerun(train_args) @@ -764,7 +770,7 @@ def testFileLists(self): remove_logs() remove_ckpt(pretrain_arguments["output_dir"]) train_args["unified_checkpoint_config"] = "skip_save_model_weight" - self.runfrist(train_args) + self.runfirst(train_args) unsave_filelists = [ "master_weights-00001-of-00002.safetensors", "master_weights-00002-of-00002.safetensors", @@ -791,7 +797,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -812,7 +818,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -831,7 +837,7 @@ def setUp(self): self.need_allclose = False - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O1" self.run_n2c4(self.run_pretrain_file, **train_args) @@ -852,7 +858,7 @@ def setUp(self): self.need_allclose = False - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O2" self.run_n2c4(self.run_pretrain_file, **train_args) @@ -869,7 +875,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["unified_checkpoint"] = 0 self.run_n2c4(self.run_pretrain_file, **train_args) @@ -889,7 +895,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -912,7 +918,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) move_checkpoint_N1C8_to_N2C4() @@ -940,7 +946,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O1" self.run_n1c8(self.run_pretrain_file, **train_args) move_checkpoint_N1C8_to_N2C4() @@ -970,7 +976,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O2" self.run_n1c8(self.run_pretrain_file, **train_args) move_checkpoint_N1C8_to_N2C4() @@ -998,7 +1004,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) move_checkpoint_N1C8_to_N2C4() @@ -1026,7 +1032,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_pretrain_file, **train_args) move_checkpoint_N2C4_to_N1C8() @@ -1054,7 +1060,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O1" self.run_n2c4(self.run_pretrain_file, **train_args) move_checkpoint_N2C4_to_N1C8() @@ -1084,7 +1090,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): train_args["fp16_opt_level"] = "O2" self.run_n2c4(self.run_pretrain_file, **train_args) move_checkpoint_N2C4_to_N1C8() @@ -1112,7 +1118,7 @@ def setUp(self): self.rtol = 1e-4 self.k = MAX_CONVERT_CONFIGS # max: 16, min: 1 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n2c4(self.run_pretrain_file, **train_args) move_checkpoint_N2C4_to_N1C8() @@ -1126,6 +1132,7 @@ def rerun(self, train_args): np.testing.assert_allclose(res[0], res[-1], rtol=self.rtol) +@pytest.mark.skipif(True, reason="Skip for None CE") class TestUnifiedCheckpointOnN1C8EnableAll(TestUnifiedCheckpointBase): def setUp(self): super().setUp() @@ -1136,7 +1143,7 @@ def setUp(self): self.need_allclose = True self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, **train_args) def rerun(self, train_args): @@ -1156,7 +1163,7 @@ def setUp(self): self.need_allclose = False self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, log_dir="log_uc", **train_args) def rerun(self, train_args): @@ -1175,7 +1182,7 @@ def setUp(self): self.need_allclose = False self.rtol = 1e-7 - def runfrist(self, train_args): + def runfirst(self, train_args): self.run_n1c8(self.run_pretrain_file, log_dir="log_pd", **train_args) def rerun(self, train_args):