Skip to content

Commit

Permalink
double accessor and show_scale (#41943) (#42014)
Browse files Browse the repository at this point in the history 
* double accessor and show_scale

* double accessor and show_scale

* rename

* fix bug in pslib config

* add unittest
  • Loading branch information
@esythan
esythan authored Apr 21, 2022
1 parent 9710469 commit efaef31
Show file tree
Hide file tree
Showing 10 changed files with 192 additions and 167 deletions.
2 changes: 1 addition & 1 deletion paddle/fluid/distributed/ps/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Table: for param storage and update

ValueAccessor: for pull param and push gradient
-----CtrCommonAccessor: pull/push value with show/click, float type
-----DownpourCtrDoubleAccessor: same as CtrCommonAccessor, other than show/click with double type
-----CtrDoubleAccessor: same as CtrCommonAccessor, other than show/click with double type
-----SparseAccessor: used for common embedding, pull value without show/click, push value with show/click
-----CommMergeAccessor: used for dense table only, for get param dim

Expand Down
3 changes: 1 addition & 2 deletions paddle/fluid/distributed/ps/table/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@ set_source_files_properties(ctr_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUT
set_source_files_properties(sparse_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
set_source_files_properties(memory_sparse_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
cc_library(sparse_sgd_rule SRCS sparse_sgd_rule.cc DEPS ${TABLE_DEPS} ps_framework_proto)
cc_library(ctr_double_accessor SRCS ctr_double_accessor.cc DEPS ${TABLE_DEPS} ps_framework_proto sparse_sgd_rule)
cc_library(ctr_accessor SRCS ctr_accessor.cc sparse_accessor.cc DEPS ${TABLE_DEPS} ps_framework_proto sparse_sgd_rule)
cc_library(ctr_accessor SRCS ctr_accessor.cc ctr_double_accessor.cc sparse_accessor.cc DEPS ${TABLE_DEPS} ps_framework_proto sparse_sgd_rule)
cc_library(memory_sparse_table SRCS memory_sparse_table.cc DEPS ps_framework_proto ${TABLE_DEPS} fs afs_wrapper ctr_accessor common_table)

set_source_files_properties(memory_sparse_geo_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
Expand Down
9 changes: 9 additions & 0 deletions paddle/fluid/distributed/ps/table/ctr_accessor.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ int CtrCommonAccessor::Initialize() {
common_feature_value.embedx_sgd_dim = _embedx_sgd_rule->Dim();
_show_click_decay_rate = _config.ctr_accessor_param().show_click_decay_rate();

if (_config.ctr_accessor_param().show_scale()) {
_show_scale = true;
}

InitAccessorInfo();
return 0;
}
Expand Down Expand Up @@ -233,6 +237,11 @@ int32_t CtrCommonAccessor::Update(float** update_values,
push_click * _config.ctr_accessor_param().click_coeff();
update_value[common_feature_value.UnseenDaysIndex()] = 0;
// TODO(zhaocaibei123): add configure show_scale
if (!_show_scale) {
push_show = 1;
}
VLOG(3) << "accessor show scale:" << _show_scale
<< ", push_show:" << push_show;
_embed_sgd_rule->UpdateValue(
update_value + common_feature_value.EmbedWIndex(),
update_value + common_feature_value.EmbedG2SumIndex(),
Expand Down
217 changes: 110 additions & 107 deletions paddle/fluid/distributed/ps/table/ctr_double_accessor.cc
View file

Large diffs are not rendered by default.

97 changes: 43 additions & 54 deletions paddle/fluid/distributed/ps/table/ctr_double_accessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
namespace paddle {
namespace distributed {

class DownpourCtrDoubleAccessor : public ValueAccessor {
class CtrDoubleAccessor : public ValueAccessor {
public:
struct DownpourCtrDoubleFeatureValue {
struct CtrDoubleFeatureValue {
/*
float unseen_days;
float delta_score;
Expand All @@ -45,60 +45,56 @@ class DownpourCtrDoubleAccessor : public ValueAccessor {
}
static int UnseenDaysIndex() { return 0; }
static int DeltaScoreIndex() {
return DownpourCtrDoubleFeatureValue::UnseenDaysIndex() + 1;
return CtrDoubleFeatureValue::UnseenDaysIndex() + 1;
}
static int ShowIndex() {
return DownpourCtrDoubleFeatureValue::DeltaScoreIndex() + 1;
return CtrDoubleFeatureValue::DeltaScoreIndex() + 1;
}
// show is double
static int ClickIndex() {
return DownpourCtrDoubleFeatureValue::ShowIndex() + 2;
}
static int ClickIndex() { return CtrDoubleFeatureValue::ShowIndex() + 2; }
// click is double
static int EmbedWIndex() {
return DownpourCtrDoubleFeatureValue::ClickIndex() + 2;
}
static int EmbedWIndex() { return CtrDoubleFeatureValue::ClickIndex() + 2; }
static int EmbedG2SumIndex() {
return DownpourCtrDoubleFeatureValue::EmbedWIndex() + 1;
return CtrDoubleFeatureValue::EmbedWIndex() + 1;
}
static int SlotIndex() {
return DownpourCtrDoubleFeatureValue::EmbedG2SumIndex() + 1;
return CtrDoubleFeatureValue::EmbedG2SumIndex() + 1;
}
static int EmbedxG2SumIndex() {
return DownpourCtrDoubleFeatureValue::SlotIndex() + 1;
return CtrDoubleFeatureValue::SlotIndex() + 1;
}
static int EmbedxWIndex() {
return DownpourCtrDoubleFeatureValue::EmbedxG2SumIndex() + 1;
return CtrDoubleFeatureValue::EmbedxG2SumIndex() + 1;
}
static float& UnseenDays(float* val) {
return val[DownpourCtrDoubleFeatureValue::UnseenDaysIndex()];
return val[CtrDoubleFeatureValue::UnseenDaysIndex()];
}
static float& DeltaScore(float* val) {
return val[DownpourCtrDoubleFeatureValue::DeltaScoreIndex()];
return val[CtrDoubleFeatureValue::DeltaScoreIndex()];
}
static double& Show(float* val) {
return ((double*)(val + DownpourCtrDoubleFeatureValue::ShowIndex()))[0];
return ((double*)(val + CtrDoubleFeatureValue::ShowIndex()))[0];
}
static double& Click(float* val) {
return ((double*)(val + DownpourCtrDoubleFeatureValue::ClickIndex()))[0];
return ((double*)(val + CtrDoubleFeatureValue::ClickIndex()))[0];
}
static float& Slot(float* val) {
return val[DownpourCtrDoubleFeatureValue::SlotIndex()];
return val[CtrDoubleFeatureValue::SlotIndex()];
}
static float& EmbedW(float* val) {
return val[DownpourCtrDoubleFeatureValue::EmbedWIndex()];
return val[CtrDoubleFeatureValue::EmbedWIndex()];
}
static float& EmbedG2Sum(float* val) {
return val[DownpourCtrDoubleFeatureValue::EmbedG2SumIndex()];
return val[CtrDoubleFeatureValue::EmbedG2SumIndex()];
}
static float& EmbedxG2Sum(float* val) {
return val[DownpourCtrDoubleFeatureValue::EmbedxG2SumIndex()];
return val[CtrDoubleFeatureValue::EmbedxG2SumIndex()];
}
static float* EmbedxW(float* val) {
return (val + DownpourCtrDoubleFeatureValue::EmbedxWIndex());
return (val + CtrDoubleFeatureValue::EmbedxWIndex());
}
};
struct DownpourCtrDoublePushValue {
struct CtrDoublePushValue {
/*
float slot;
float show;
Expand All @@ -110,35 +106,27 @@ class DownpourCtrDoubleAccessor : public ValueAccessor {
static int DimSize(int dim, int embedx_dim) { return sizeof(float); }
static int Size(int embedx_dim) { return Dim(embedx_dim) * sizeof(float); }
static int SlotIndex() { return 0; }
static int ShowIndex() {
return DownpourCtrDoublePushValue::SlotIndex() + 1;
}
static int ClickIndex() {
return DownpourCtrDoublePushValue::ShowIndex() + 1;
}
static int EmbedGIndex() {
return DownpourCtrDoublePushValue::ClickIndex() + 1;
}
static int EmbedxGIndex() {
return DownpourCtrDoublePushValue::EmbedGIndex() + 1;
}
static int ShowIndex() { return CtrDoublePushValue::SlotIndex() + 1; }
static int ClickIndex() { return CtrDoublePushValue::ShowIndex() + 1; }
static int EmbedGIndex() { return CtrDoublePushValue::ClickIndex() + 1; }
static int EmbedxGIndex() { return CtrDoublePushValue::EmbedGIndex() + 1; }
static float& Slot(float* val) {
return val[DownpourCtrDoublePushValue::SlotIndex()];
return val[CtrDoublePushValue::SlotIndex()];
}
static float& Show(float* val) {
return val[DownpourCtrDoublePushValue::ShowIndex()];
return val[CtrDoublePushValue::ShowIndex()];
}
static float& Click(float* val) {
return val[DownpourCtrDoublePushValue::ClickIndex()];
return val[CtrDoublePushValue::ClickIndex()];
}
static float& EmbedG(float* val) {
return val[DownpourCtrDoublePushValue::EmbedGIndex()];
return val[CtrDoublePushValue::EmbedGIndex()];
}
static float* EmbedxG(float* val) {
return val + DownpourCtrDoublePushValue::EmbedxGIndex();
return val + CtrDoublePushValue::EmbedxGIndex();
}
};
struct DownpourCtrDoublePullValue {
struct CtrDoublePullValue {
/*
float show;
float click;
Expand All @@ -153,20 +141,20 @@ class DownpourCtrDoubleAccessor : public ValueAccessor {
static int EmbedWIndex() { return 2; }
static int EmbedxWIndex() { return 3; }
static float& Show(float* val) {
return val[DownpourCtrDoublePullValue::ShowIndex()];
return val[CtrDoublePullValue::ShowIndex()];
}
static float& Click(float* val) {
return val[DownpourCtrDoublePullValue::ClickIndex()];
return val[CtrDoublePullValue::ClickIndex()];
}
static float& EmbedW(float* val) {
return val[DownpourCtrDoublePullValue::EmbedWIndex()];
return val[CtrDoublePullValue::EmbedWIndex()];
}
static float* EmbedxW(float* val) {
return val + DownpourCtrDoublePullValue::EmbedxWIndex();
return val + CtrDoublePullValue::EmbedxWIndex();
}
};
DownpourCtrDoubleAccessor() {}
virtual ~DownpourCtrDoubleAccessor() {}
CtrDoubleAccessor() {}
virtual ~CtrDoubleAccessor() {}
virtual int Initialize();
// 初始化AccessorInfo
virtual void InitAccessorInfo();
Expand All @@ -182,7 +170,7 @@ class DownpourCtrDoubleAccessor : public ValueAccessor {
// update delta_score and unseen_days after save
virtual void UpdateStatAfterSave(float* value, int param) override;
// 判断该value是否保存到ssd
virtual bool save_ssd(float* value);
virtual bool SaveSSD(float* value);
// virtual bool save_cache(float* value, int param, double
// global_cache_threshold) override;
// keys不存在时，为values生成随机值
Expand All @@ -206,14 +194,14 @@ class DownpourCtrDoubleAccessor : public ValueAccessor {
virtual float GetField(float* value, const std::string& name) override {
CHECK(name == "show");
if (name == "show") {
return (float)DownpourCtrDoubleFeatureValue::Show(value);
return (float)CtrDoubleFeatureValue::Show(value);
}
return 0.0;
}
// DEFINE_GET_INDEX(DownpourCtrDoubleFeatureValue, show)
// DEFINE_GET_INDEX(DownpourCtrDoubleFeatureValue, click)
// DEFINE_GET_INDEX(DownpourCtrDoubleFeatureValue, embed_w)
// DEFINE_GET_INDEX(DownpourCtrDoubleFeatureValue, embedx_w)
// DEFINE_GET_INDEX(CtrDoubleFeatureValue, show)
// DEFINE_GET_INDEX(CtrDoubleFeatureValue, click)
// DEFINE_GET_INDEX(CtrDoubleFeatureValue, embed_w)
// DEFINE_GET_INDEX(CtrDoubleFeatureValue, embedx_w)
private:
double ShowClickScore(double show, double click);

Expand All @@ -222,6 +210,7 @@ class DownpourCtrDoubleAccessor : public ValueAccessor {
SparseValueSGDRule* _embedx_sgd_rule;
float _show_click_decay_rate;
int32_t _ssd_unseenday_threshold;
bool _show_scale = false;
};
} // namespace distributed
} // namespace paddle
2 changes: 2 additions & 0 deletions paddle/fluid/distributed/ps/table/table.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "paddle/fluid/distributed/ps/table/memory_dense_table.h"

#include "paddle/fluid/distributed/ps/table/ctr_accessor.h"
#include "paddle/fluid/distributed/ps/table/ctr_double_accessor.h"
#include "paddle/fluid/distributed/ps/table/memory_sparse_geo_table.h"
#include "paddle/fluid/distributed/ps/table/memory_sparse_table.h"
#include "paddle/fluid/distributed/ps/table/sparse_accessor.h"
Expand All @@ -39,6 +40,7 @@ REGISTER_PSCORE_CLASS(Table, MemorySparseTable);
REGISTER_PSCORE_CLASS(Table, MemorySparseGeoTable);
REGISTER_PSCORE_CLASS(ValueAccessor, CommMergeAccessor);
REGISTER_PSCORE_CLASS(ValueAccessor, CtrCommonAccessor);
REGISTER_PSCORE_CLASS(ValueAccessor, CtrDoubleAccessor);
REGISTER_PSCORE_CLASS(ValueAccessor, SparseAccessor);
REGISTER_PSCORE_CLASS(SparseValueSGDRule, StdAdaGradSGDRule);
REGISTER_PSCORE_CLASS(SparseValueSGDRule, SparseAdamSGDRule);
Expand Down
1 change: 1 addition & 0 deletions paddle/fluid/distributed/the_one_ps.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ message CtrAccessorParameter {
// will be delete in shrink_model
optional int32 ssd_unseenday_threshold = 9
[ default = 1 ]; // threshold to save ssd
optional bool show_scale = 10 [ default = true ];
}

message TensorAccessorParameter {
Expand Down
1 change: 1 addition & 0 deletions paddle/fluid/framework/distributed_strategy.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ message CtrAccessorParameter {
[ default = 0.8 ]; // threshold to shrink a feasign
optional float delete_after_unseen_days = 8 [ default = 30 ];
optional int32 ssd_unseenday_threshold = 9 [ default = 1 ];
optional bool show_scale = 10 [ default = true ];
}

message TableAccessorSaveParameter {
Expand Down
14 changes: 11 additions & 3 deletions python/paddle/distributed/fleet/base/distributed_strategy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,19 +611,27 @@ def set_sparse_table_config(table_data, config):
"DownpourCtrAccessor")
if accessor_class not in support_sparse_accessor_class:
raise ValueError(
"support sparse_accessor_class: [''DownpourSparseValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', 'DownpourDoubleUnitAccessor'], but actual %s"
"support sparse_accessor_class: ['DownpourSparseValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', 'DownpourDoubleUnitAccessor'], but actual %s"
% (accessor_class))

if configs.get("use_cvm", True):
table_data.accessor.accessor_class = 'CtrCommonAccessor'
if accessor_class.find("Double") >= 0:
table_data.accessor.accessor_class = 'CtrDoubleAccessor'
else:
table_data.accessor.accessor_class = 'CtrCommonAccessor'

if not configs.get("use_cvm", True):
table_data.accessor.accessor_class = 'SparseAccessor'

table_data.accessor.embedx_dim = config.get('sparse_embedx_dim', 8)
table_data.accessor.fea_dim = table_data.accessor.embedx_dim + 3
table_data.accessor.embedx_threshold = config.get(
'sparse_embedx_threshold', 10)

if accessor_class == 'DownpourUnitAccessor':
table_data.accessor.ctr_accessor_param.show_scale = False
else:
table_data.accessor.ctr_accessor_param.show_scale = True

table_data.accessor.ctr_accessor_param.nonclk_coeff = config.get(
'sparse_nonclk_coeff', 0.1)
table_data.accessor.ctr_accessor_param.click_coeff = config.get(
Expand Down
13 changes: 13 additions & 0 deletions python/paddle/fluid/tests/unittests/test_fleet_distributed_strategy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,9 +310,22 @@ def test_fleet_desc_configs(self):
"embed_sparse_optimizer": "std_adagrad"
}
strategy.fleet_desc_configs = configs
self.assertEqual(strategy.sparse_table_configs[0]
.accessor.ctr_accessor_param.show_scale, False)
self.assertEqual(strategy.sparse_table_configs[0]
.accessor.embed_sgd_param.adagrad.initial_range, 0)

strategy = paddle.distributed.fleet.DistributedStrategy()
configs = {}
configs['emb'] = {
"sparse_accessor_class": "DownpourCtrDoubleAccessor",
"embed_sparse_optimizer": "std_adagrad"
}
strategy.fleet_desc_configs = configs
self.assertEqual(strategy.sparse_table_configs[0]
.accessor.embed_sgd_param.adagrad.initial_range,
0.0001)

def test_trainer_desc_configs(self):
strategy = paddle.distributed.fleet.DistributedStrategy()
configs = {
Expand Down

0 comments on commit efaef31

Please sign in to comment.