Soft Margin loss (#27660)

CarMiranda · facebook-github-bot · commit 9540f6c3fe33 · 2019-10-16T12:04:08.000-07:00
Summary: In accordance with #25883, I added the `SoftMarginLoss` module and `soft_margin_loss` functional. Pull Request resolved: #27660 Differential Revision: D17958325 Pulled By: yf225 fbshipit-source-id: c14422765e6e1fdabf6c9687080e6d5ff490d300
diff --git a/test/cpp/api/functional.cpp b/test/cpp/api/functional.cpp
@@ -73,6 +73,19 @@ TEST_F(FunctionalTest, CosineSimilarity) {
   ASSERT_TRUE(output.allclose(expected, 1e-04));
 }
 
+TEST_F(FunctionalTest, SoftMarginLossDefaultOptions) {
+  auto input = torch::tensor({2., 4., 1., 3.}, torch::requires_grad());
+  auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat);
+  auto output =
+      F::soft_margin_loss(input, target);
+  auto expected = torch::tensor({1.3767317}, torch::kFloat);
+  auto s = output.sum();
+  s.backward();
+
+  ASSERT_TRUE(output.allclose(expected));
+  ASSERT_EQ(input.sizes(), input.grad().sizes());
+}
+
 TEST_F(FunctionalTest, MultiLabelSoftMarginLossDefaultOptions) {
   auto input = torch::tensor({{0., 2., 2., 0.}, {2., 1., 0., 1.}}, torch::requires_grad());
   auto target = torch::tensor({{0., 0., 1., 0.}, {1., 0., 1., 1.}}, torch::kFloat);
@@ -86,6 +99,19 @@ TEST_F(FunctionalTest, MultiLabelSoftMarginLossDefaultOptions) {
   ASSERT_EQ(input.sizes(), input.grad().sizes());
 }
 
+TEST_F(FunctionalTest, SoftMarginLossNoReduction) {
+  auto input = torch::tensor({2., 4., 1., 3.}, torch::requires_grad());
+  auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat);
+  auto output =
+      F::soft_margin_loss(input, target, torch::Reduction::None);
+  auto expected = torch::tensor({2.1269281, 0.01814993, 0.3132617, 3.0485873}, torch::kFloat);
+  auto s = output.sum();
+  s.backward();
+
+  ASSERT_TRUE(output.allclose(expected));
+  ASSERT_EQ(input.sizes(), input.grad().sizes());
+}
+
 TEST_F(FunctionalTest, MultiLabelSoftMarginLossWeightedNoReduction) {
   auto input = torch::tensor({{0., 2., 2., 0.}, {2., 1., 0., 1.}}, torch::requires_grad());
   auto target = torch::tensor({{0., 0., 1., 0.}, {1., 0., 1., 1.}}, torch::kFloat);
diff --git a/test/cpp/api/modules.cpp b/test/cpp/api/modules.cpp
@@ -1096,6 +1096,19 @@ TEST_F(ModulesTest, CosineSimilarity) {
   ASSERT_EQ(input1.sizes(), input1.grad().sizes());
 }
 
+TEST_F(ModulesTest, SoftMarginLossDefaultOptions) {
+  SoftMarginLoss loss;
+  auto input = torch::tensor({2., 4., 1., 3.}, torch::requires_grad());
+  auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat);
+  auto output = loss->forward(input, target);
+  auto expected = torch::tensor({1.3767317}, torch::kFloat);
+  auto s = output.sum();
+  s.backward();
+
+  ASSERT_TRUE(output.allclose(expected));
+  ASSERT_EQ(input.sizes(), input.grad().sizes());
+}
+
 TEST_F(ModulesTest, MultiLabelSoftMarginLossDefaultOptions) {
   MultiLabelSoftMarginLoss loss;
   auto input = torch::tensor({{0., 2., 2., 0.}, {2., 1., 0., 1.}}, torch::requires_grad());
@@ -1109,6 +1122,19 @@ TEST_F(ModulesTest, MultiLabelSoftMarginLossDefaultOptions) {
   ASSERT_EQ(input.sizes(), input.grad().sizes());
 }
 
+TEST_F(ModulesTest, SoftMarginLossNoReduction) {
+  SoftMarginLoss loss(torch::Reduction::None);
+  auto input = torch::tensor({2., 4., 1., 3.}, torch::requires_grad());
+  auto target = torch::tensor({-1., 1., 1., -1.}, torch::kFloat);
+  auto output = loss->forward(input, target);
+  auto expected = torch::tensor({2.1269281, 0.01814993, 0.3132617, 3.0485873}, torch::kFloat);
+  auto s = output.sum();
+  s.backward();
+
+  ASSERT_TRUE(output.allclose(expected));
+  ASSERT_EQ(input.sizes(), input.grad().sizes());
+}
+
 TEST_F(ModulesTest, MultiLabelSoftMarginLossWeightedNoReduction) {
   auto input = torch::tensor({{0., 2., 2., 0.}, {2., 1., 0., 1.}}, torch::requires_grad());
   auto target = torch::tensor({{0., 0., 1., 0.}, {1., 0., 1., 1.}}, torch::kFloat);
@@ -1733,6 +1759,10 @@ TEST_F(ModulesTest, PrettyPrintMultiLabelSoftMarginLoss) {
   ASSERT_EQ(c10::str(MultiLabelSoftMarginLoss()), "torch::nn::MultiLabelSoftMarginLoss()");
 }
 
+TEST_F(ModulesTest, PrettyPrintSoftMarginLoss) {
+  ASSERT_EQ(c10::str(SoftMarginLoss()), "torch::nn::SoftMarginLoss()");
+}
+
 TEST_F(ModulesTest, PrettyPrintCosineSimilarity) {
   ASSERT_EQ(
       c10::str(CosineSimilarity()),
diff --git a/torch/csrc/api/include/torch/nn/functional/loss.h b/torch/csrc/api/include/torch/nn/functional/loss.h
@@ -45,6 +45,13 @@ inline Tensor cosine_embedding_loss(
       input1, input2, target, options.margin(), options.reduction());
 }
 
+inline Tensor soft_margin_loss(
+    const Tensor& input,
+    const Tensor& target,
+    const SoftMarginLossOptions& options = {}) {
+  return torch::soft_margin_loss(input, target, options.reduction());
+}
+
 inline Tensor multilabel_soft_margin_loss(
     const Tensor& input,
     const Tensor& target,
diff --git a/torch/csrc/api/include/torch/nn/modules/loss.h b/torch/csrc/api/include/torch/nn/modules/loss.h
@@ -125,14 +125,39 @@ TORCH_MODULE(CosineEmbeddingLoss);
 
 // ============================================================================
 
+/// Creates a criterion that optimizes a two-class classification
+/// logistic loss between input tensor :math:`x` and target tensor :math:`y`
+/// (containing 1 or -1).
+struct TORCH_API SoftMarginLossImpl : public Cloneable<SoftMarginLossImpl> {
+  explicit SoftMarginLossImpl(const SoftMarginLossOptions& options_ = {});
+
+  /// Pretty prints the `SoftMarginLoss` module into the given `stream`.
+  void pretty_print(std::ostream& stream) const override;
+
+  void reset() override;
+
+  Tensor forward(const Tensor& input, const Tensor& target);
+
+  /// The options with which this `Module` was constructed.
+  SoftMarginLossOptions options;
+};
+
+/// A `ModuleHolder` subclass for `SoftMarginLossImpl`.
+/// See the documentation for `SoftMarginLossImpl` class to learn what methods it
+/// provides, or the documentation for `ModuleHolder` to learn about PyTorch's
+/// module storage semantics.
+TORCH_MODULE(SoftMarginLoss);
+
+// ============================================================================
+
 /// Creates a criterion that optimizes a multi-label one-versus-all
 /// loss based on max-entropy, between input :math:`x` and target :math:`y` of size
 /// :math:`(N, C)`.
 struct TORCH_API MultiLabelSoftMarginLossImpl : public Cloneable<MultiLabelSoftMarginLossImpl> {
   explicit MultiLabelSoftMarginLossImpl(
     const MultiLabelSoftMarginLossOptions& options_ = {});
 
-  /// Pretty prints the `L1Loss` module into the given `stream`.
+  /// Pretty prints the `MultiLabelSoftMarginLoss` module into the given `stream`.
   void pretty_print(std::ostream& stream) const override;
 
   void reset() override;
diff --git a/torch/csrc/api/include/torch/nn/options/loss.h b/torch/csrc/api/include/torch/nn/options/loss.h
@@ -61,6 +61,20 @@ struct TORCH_API CosineEmbeddingLossOptions {
 
 // ============================================================================
 
+/// Options for a soft margin loss functional and module.
+struct TORCH_API SoftMarginLossOptions {
+  SoftMarginLossOptions(torch::Reduction::Reduction reduction = torch::Reduction::Mean)
+    : reduction_(reduction) {}
+
+  /// Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'.
+  /// 'none': no reduction will be applied, 'mean': the sum of the output will
+  /// be divided by the number of elements in the output, 'sum': the output will
+  /// be summed. Default: 'mean'
+  TORCH_ARG(torch::Reduction::Reduction, reduction);
+};
+
+// ============================================================================
+
 /// Options for a multi-label soft margin loss functional and module.
 struct TORCH_API MultiLabelSoftMarginLossOptions {
   /// A manual rescaling weight given to each
diff --git a/torch/csrc/api/src/nn/modules/loss.cpp b/torch/csrc/api/src/nn/modules/loss.cpp
@@ -126,5 +126,20 @@ Tensor TripletMarginLossImpl::forward(
   return F::triplet_margin_loss(anchor, positive, negative, options);
 }
 
+// ============================================================================
+
+SoftMarginLossImpl::SoftMarginLossImpl(
+    const torch::nn::SoftMarginLossOptions& options_) : options(options_) {}
+
+void SoftMarginLossImpl::reset() {}
+
+void SoftMarginLossImpl::pretty_print(std::ostream& stream) const {
+  stream << "torch::nn::SoftMarginLoss()";
+}
+
+Tensor SoftMarginLossImpl::forward(const Tensor& input, const Tensor& target) {
+  return F::soft_margin_loss(input, target, options);
+}
+
 } // namespace nn
 } // namespace torch
