Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support log sigmoid gradient #4719

Merged
merged 3 commits into from
Aug 7, 2020
Merged

Support log sigmoid gradient #4719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd617de
add log's gradient op and its related gradient test
zhijxu-MS Aug 4, 2020
458d2e8
support sigmoid's gradient op
zhijxu-MS Aug 5, 2020
d806151
resolve review comments
zhijxu-MS Aug 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions orttraining/orttraining/core/graph/gradient_builder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,13 @@ IMPLEMENT_GRADIENT_BUILDER(GetSinGradient) {
{GI(0)})};
}

IMPLEMENT_GRADIENT_BUILDER(GetLogGradient) {
return std::vector<NodeDef>{
NodeDef("Div",
{GO(0), I(0)},
{GI(0)})};
}

IMPLEMENT_GRADIENT_BUILDER(GetTanhGradient) {
return std::vector<NodeDef>{
NodeDef("TanhGrad",
Expand Down Expand Up @@ -712,6 +719,20 @@ IMPLEMENT_GRADIENT_BUILDER(GetConvGradient) {
SrcNodeAttributes())};
}

IMPLEMENT_GRADIENT_BUILDER(GetSigmoidGradient) {
auto const_one = OneConstantNode();
return std::vector<NodeDef>{
const_one,
NodeDef("Sub",
{const_one.output_args[0], O(0)},
{IA("one_minus_output")}),
NodeDef("Mul",
{IA("one_minus_output"), O(0)},
{IA("sigmoid_derivate")}),
NodeDef("Mul",
{IA("sigmoid_derivate"), GO(0)},
{GI(0)})};
}
IMPLEMENT_GRADIENT_BUILDER(GetSoftmaxGradient) {
return std::vector<NodeDef>{
NodeDef(OpDef{"SoftmaxGrad", kMSDomain, 1},
Expand Down
2 changes: 2 additions & 0 deletions orttraining/orttraining/core/graph/gradient_builder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ namespace training {

DECLARE_GRADIENT_BUILDER(GetCastGradient)
DECLARE_GRADIENT_BUILDER(GetSinGradient)
DECLARE_GRADIENT_BUILDER(GetLogGradient)
DECLARE_GRADIENT_BUILDER(GetTanhGradient)
DECLARE_GRADIENT_BUILDER(GetSqrtGradient)
DECLARE_GRADIENT_BUILDER(GetErfGradient)
Expand All @@ -38,6 +39,7 @@ DECLARE_GRADIENT_BUILDER(GetGatherGradient)
DECLARE_GRADIENT_BUILDER(GetConvGradient)
DECLARE_GRADIENT_BUILDER(GetUnsqueezeGradient)
DECLARE_GRADIENT_BUILDER(GetSqueezeGradient)
DECLARE_GRADIENT_BUILDER(GetSigmoidGradient)
DECLARE_GRADIENT_BUILDER(GetSoftmaxGradient)
DECLARE_GRADIENT_BUILDER(GetLogSoftmaxGradient)
DECLARE_GRADIENT_BUILDER(GetSoftmaxCrossEntropyGradient)
Expand Down
2 changes: 2 additions & 0 deletions orttraining/orttraining/core/graph/gradient_builder_registry.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ void GradientBuilderRegistry::RegisterGradientBuilders() {
// Register gradient builders here.
REGISTER_GRADIENT_BUILDER("Cast", GetCastGradient);
REGISTER_GRADIENT_BUILDER("Sin", GetSinGradient);
REGISTER_GRADIENT_BUILDER("Log", GetLogGradient);
REGISTER_GRADIENT_BUILDER("Tanh", GetTanhGradient);
REGISTER_GRADIENT_BUILDER("Sqrt", GetSqrtGradient);
REGISTER_GRADIENT_BUILDER("Erf", GetErfGradient);
Expand All @@ -69,6 +70,7 @@ void GradientBuilderRegistry::RegisterGradientBuilders() {
REGISTER_GRADIENT_BUILDER("Conv", GetConvGradient);
REGISTER_GRADIENT_BUILDER("Squeeze", GetSqueezeGradient);
REGISTER_GRADIENT_BUILDER("Unsqueeze", GetUnsqueezeGradient);
REGISTER_GRADIENT_BUILDER("Sigmoid", GetSigmoidGradient);
REGISTER_GRADIENT_BUILDER("Softmax", GetSoftmaxGradient);
REGISTER_GRADIENT_BUILDER("LogSoftmax", GetLogSoftmaxGradient);
REGISTER_GRADIENT_BUILDER("SoftmaxCrossEntropy", GetSoftmaxCrossEntropyGradient);
Expand Down
20 changes: 20 additions & 0 deletions orttraining/orttraining/test/gradient/gradient_ops_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,22 @@ TEST(GradientCheckerTest, SinGrad) {
UnaryOpGradientTest("Sin");
}

TEST(GradientCheckerTest, LogGrad) {
TensorShape shape({2,5,6});

std::function<float(float)> transformer = [](float x) { return std::fabs(x) + 1e-1f; };
TensorInfo x_info{shape, true, &transformer};

float max_error;
float error_tolerance = 1e-3f;
GradientChecker<float, float, float> gradient_checker;
OpDef op_def{"Log"};

gradient_checker.ComputeGradientError(op_def, {x_info}, {shape}, &max_error);

EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

TEST(GradientCheckerTest, TanhGrad) {
UnaryOpGradientTest("Tanh");
}
Expand Down Expand Up @@ -1141,6 +1157,10 @@ TEST(GradientCheckerTest, DISABLED_BatchNormalizationGrad) {
}
#endif

TEST(GradientCheckerTest, SigmoidGrad) {
UnaryOpGradientTest("Sigmoid");
}

void GradientCheckerSoftmaxGradHelper(bool is_log_softmax) {
TensorShape shape({3, 4, 5});
float max_error;
Expand Down