Merge pull request PaddlePaddle#23 from jim19930609/merge

Rearranged Eager Dygraph directory and files

paddle/fluid/eager/CMakeLists.txt

@@ -1,21 +1,17 @@
-add_subdirectory(nodes)
-add_subdirectory(api)
-add_subdirectory(autocodegen)
-add_subdirectory(legacy)
-
-
-set(eager_deps eager_api utils function_api backward autograd_meta grad_node_info legacy pten pten_api)
+set(eager_deps eager_api backward utils autograd_meta grad_node_info legacy pten pten_api)
 set(fluid_deps tracer layer proto_desc operator op_registry variable_helper memcpy)
 set(generated_deps dygraph_function dygraph_node)
 
-add_subdirectory(generated)
-add_subdirectory(tests)
+add_subdirectory(auto_code_generator)
+add_subdirectory(api)
+add_subdirectory(accumulation)
+add_subdirectory(legacy)
+
+cc_library(grad_node_info SRCS grad_node_info.cc DEPS gradient_accumulation pten pten_api)
 
 cc_library(autograd_meta SRCS autograd_meta.cc DEPS pten pten_api)
-cc_library(backward SRCS backward.cc DEPS autograd_meta grad_node_info)
-cc_library(gradient_accumulation SRCS gradient_accumulation.cc DEPS blas pten pten_api var_type_traits layer math_function)
-cc_library(function_api SRCS function_api.cc DEPS pten pten_api layer proto_desc operator op_registry variable_helper memcpy scale_op)
-cc_library(utils SRCS utils.cc DEPS pten pten_api layer proto_desc operator op_registry variable_helper memcpy scale_op)
+cc_library(utils SRCS utils.cc DEPS pten pten_api layer proto_desc operator op_registry variable_helper memcpy scale_op accumulation_node autograd_meta)
+cc_library(legacy SRCS ${DYGRAPH_LEGACY} DEPS global_utils proto_desc operator pten pten_api op_registry variable_helper memcpy) 
+cc_library(backward SRCS backward.cc DEPS utils autograd_meta grad_node_info)
 
-cc_library(grad_node_info SRCS grad_node_info.cc ${DYGRAPH_BACKWARD_NODES} DEPS function_api pten pten_api gradient_accumulation)
-cc_library(legacy SRCS ${DYGRAPH_LEGACY} DEPS proto_desc operator pten pten_api op_registry variable_helper memcpy)
+add_subdirectory(tests)

paddle/fluid/eager/accumulation/CMakeLists.txt

@@ -0,0 +1,2 @@
+cc_library(accumulation_node SRCS accumulation_node.cc DEPS pten pten_api grad_node_info)
+cc_library(gradient_accumulation SRCS gradient_accumulation.cc DEPS blas pten pten_api var_type_traits layer math_function)

paddle/fluid/eager/nodes/accumulation_node.cc → paddle/fluid/eager/accumulation/accumulation_node.cc

@@ -12,15 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/eager/nodes/accumulation_node.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/accumulation/gradient_accumulation.h"
 #include "paddle/fluid/eager/eager_tensor.h"
-#include "paddle/fluid/eager/function_api.h"
-#include "paddle/fluid/eager/gradient_accumulation.h"
-#include "paddle/fluid/platform/device_context.h"
+
 #include "paddle/pten/api/all.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/include/core.h"
 
+#include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/errors.h"
 

paddle/fluid/eager/gradient_accumulation.cc → paddle/fluid/eager/accumulation/gradient_accumulation.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/eager/gradient_accumulation.h"
+#include "paddle/fluid/eager/accumulation/gradient_accumulation.h"
 #include <algorithm>
 #include <memory>
 #include <utility>

paddle/fluid/eager/api/CMakeLists.txt

@@ -1 +1,4 @@
-cc_library(eager_api SRCS fwd_ops_impl.cc api.cc DEPS function_api pten autograd_meta grad_node_info)
+add_subdirectory(utils)
+add_subdirectory(generated)
+
+cc_library(eager_api SRCS all.cc DEPS global_utils hook_utils eager_scale)

paddle/fluid/eager/api/all.cc

@@ -0,0 +1,18 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "paddle/fluid/eager/api/all.h"
+
+namespace egr {}  // namespace egr

paddle/fluid/eager/api/all.h

@@ -0,0 +1,19 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#pragma once
+
+#include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h"
+#include "paddle/fluid/eager/api/utils/global_utils.h"
+#include "paddle/fluid/eager/api/utils/hook_utils.h"

paddle/fluid/eager/api/generated/.gitignore

@@ -0,0 +1 @@
+fluid_generated/**

paddle/fluid/eager/api/generated/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory(eager_generated)
+add_subdirectory(fluid_generated)

paddle/fluid/eager/api/generated/eager_generated/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory(backwards)
+add_subdirectory(forwards)

paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt

@@ -0,0 +1 @@
+cc_library(scale_node SRCS scale_node.cc DEPS global_utils pten pten_api grad_node_info)

paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc

@@ -0,0 +1,158 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
+#include "paddle/fluid/eager/api/utils/global_utils.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+
+#include "paddle/pten/api/all.h"
+
+#include "paddle/fluid/platform/device_context.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/fluid/platform/errors.h"
+
+#include "glog/logging.h"
+
+namespace egr {
+
+template <typename DeviceContext>
+static void ScaleDeviceDispatch(const pten::DenseTensor& dense_tensor,
+                                const DeviceContext& dev_ctx, float scale,
+                                float bias, bool bias_after_scale,
+                                pten::DenseTensor* dense_out) {
+  switch (dense_tensor.dtype()) {
+    case pten::DataType::FLOAT64: {
+      pten::Scale<double>(dev_ctx, dense_tensor /* tensor */, scale /* scale */,
+                          bias /* bias */,
+                          bias_after_scale /* bias_after_scale */,
+                          dense_out /* out tensor */);
+      break;
+    }
+    case pten::DataType::FLOAT32: {
+      pten::Scale<float>(dev_ctx, dense_tensor /* tensor */, scale /* scale */,
+                         bias /* bias */,
+                         bias_after_scale /* bias_after_scale */,
+                         dense_out /* out tensor */);
+      break;
+    }
+    case pten::DataType::INT64: {
+      pten::Scale<int64_t>(dev_ctx, dense_tensor /* tensor */,
+                           scale /* scale */, bias /* bias */,
+                           bias_after_scale /* bias_after_scale */,
+                           dense_out /* out tensor */);
+      break;
+    }
+    case pten::DataType::INT32: {
+      pten::Scale<int32_t>(dev_ctx, dense_tensor /* tensor */,
+                           scale /* scale */, bias /* bias */,
+                           bias_after_scale /* bias_after_scale */,
+                           dense_out /* out tensor */);
+      break;
+    }
+    default: {
+      PADDLE_THROW(paddle::platform::errors::Fatal("Unsupported data type"));
+      break;
+    }
+  }
+}
+
+void ScaleAPI(const egr::EagerTensor& x, float scale, float bias,
+              bool bias_after_scale, egr::EagerTensor* out) {
+  // TODO(jiabin): Support multiple tensor here, Create DenseTensor is not a
+  // proper way to Demo it
+  // Run Forward Function
+  auto dense_tensor = std::dynamic_pointer_cast<pten::DenseTensor>(x.impl());
+  // Init output tensor
+  auto tensor_meta = pten::DenseTensorMeta(
+      dense_tensor->dtype(), dense_tensor->dims(), dense_tensor->layout());
+  auto place = dense_tensor->place();
+  size_t bytes_size = paddle::framework::product(dense_tensor->dims()) *
+                      SizeOf(dense_tensor->dtype());
+  auto dense_out = std::make_shared<pten::DenseTensor>(
+      pten::make_intrusive<paddle::experimental::SharedStorage>(
+          paddle::memory::Alloc(place, bytes_size), 0),
+      std::move(tensor_meta));
+  // Handle Device Context
+  const paddle::platform::Place& expected_kernel_place =
+      Controller::Instance().GetExpectedPlace();
+  paddle::platform::DeviceContextPool& pool =
+      paddle::platform::DeviceContextPool::Instance();
+
+  if (expected_kernel_place == paddle::platform::CPUPlace()) {
+    auto* dev_ctx = dynamic_cast<paddle::platform::CPUDeviceContext*>(
+        pool.Get(expected_kernel_place));
+    if (!dev_ctx) {
+      PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch"));
+    }
+    ScaleDeviceDispatch<paddle::platform::CPUDeviceContext>(
+        *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale,
+        dense_out.get());
+
+  } else if (expected_kernel_place == paddle::platform::CUDAPlace()) {
+    auto* dev_ctx = dynamic_cast<paddle::platform::CUDADeviceContext*>(
+        pool.Get(expected_kernel_place));
+    if (!dev_ctx) {
+      PADDLE_THROW(paddle::platform::errors::Fatal("Backend mismatch"));
+    }
+    ScaleDeviceDispatch<paddle::platform::CUDADeviceContext>(
+        *dense_tensor.get(), *dev_ctx, scale, bias, bias_after_scale,
+        dense_out.get());
+
+  } else {
+    PADDLE_THROW(paddle::platform::errors::Fatal(
+        "Only CPU and CUDA Backend are supported for now"));
+  }
+
+  out->set_impl(dense_out);
+}
+
+void GradNodeScale::SetTensorWrappers_X(
+    const std::vector<egr::EagerTensor>& tensors) {
+  // Does nothing for scale
+}
+
+void GradNodeScale::SetAttributes_scale(float scale) { scale_ = scale; }
+
+std::vector<std::vector<egr::EagerTensor>> GradNodeScale::operator()(
+    const std::vector<std::vector<egr::EagerTensor>>& grads) {
+  // 1. Check Output Size
+  PADDLE_ENFORCE(((grads.size() == 1) && (grads[0].size() == 1)),
+                 paddle::platform::errors::Fatal(
+                     "ScaleGradNode should take exactly 1 grad tensor"
+                     "However received: %d",
+                     grads.size()));
+  std::vector<std::vector<egr::EagerTensor>> outs;
+  // 2. Create needed out parttern
+  egr::EagerTensor out;
+  // Apply Gradient Hooks
+  if (GradientHooksRegistered()) {
+    // TODO(jiabin): Shall we apply hook slot by slot here or accept
+    // vector<vector<pten::tensor>> to apply all hooks?
+    std::vector<std::vector<egr::EagerTensor>> hooked_grads =
+        ApplyGradientHooks(grads);
+    ScaleAPI(/* slot by slot set */ hooked_grads[0][0], scale_, 0.0 /* bias */,
+             true /* bias_after_scale */, &out);
+  } else {
+    ScaleAPI(grads[0][0], scale_, 0.0 /* bias */, true /* bias_after_scale */,
+             &out);
+  }
+
+  // Apply Reduce Hooks
+  if (ReduceHooksRegistered()) {
+    ApplyReduceHooks();
+  }
+  return {{out}};
+}
+
+}  // namespace egr

paddle/fluid/eager/nodes/scale_node.h → paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h

@@ -27,6 +27,9 @@
 */
 namespace egr {
 
+void ScaleAPI(const egr::EagerTensor& x, float scale, float bias,
+              bool bias_after_scale, egr::EagerTensor* out);
+
 class GradNodeScale : public GradNodeBase {
  public:
   // Constructor: configure fwd input tensors to grad node

paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt

@@ -0,0 +1 @@
+cc_library(eager_scale SRCS scale.cc DEPS pten_api pten autograd_meta scale_node)

paddle/fluid/eager/api/fwd_ops_impl.cc → paddle/fluid/eager/api/generated/eager_generated/forwards/scale.cc

@@ -22,14 +22,12 @@
  * them with auto code generator later.
  * **/
 
-#include "glog/logging.h"
-
+#include "paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h"
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
 #include "paddle/fluid/eager/autograd_meta.h"
-
-#include "paddle/fluid/eager/nodes/scale_node.h"
-
 #include "paddle/fluid/eager/eager_tensor.h"
-#include "paddle/fluid/eager/function_api.h"
+#include "paddle/fluid/eager/utils.h"
+
 #include "paddle/pten/api/all.h"
 #include "paddle/pten/include/core.h"
 

paddle/fluid/eager/api/generated/eager_generated/forwards/scale.h

@@ -0,0 +1,23 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/eager/eager_tensor.h"
+namespace egr {
+
+egr::EagerTensor scale(const egr::EagerTensor& x, float scale, float bias,
+                       bool bias_after_scale, bool trace_backward);
+
+}  // namespace egr

paddle/fluid/eager/api/utils/CMakeLists.txt

@@ -0,0 +1,2 @@
+cc_library(global_utils SRCS global_utils.cc DEPS enforce)
+cc_library(hook_utils SRCS hook_utils.cc DEPS pten autograd_meta grad_node_info utils accumulation_node)

paddle/fluid/eager/api/utils/global_utils.cc

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "paddle/fluid/eager/api/utils/global_utils.h"
+
+namespace egr {
+
+Controller* Controller::controller_ = new Controller();
+
+}  // namespace egr

paddle/fluid/eager/function_api.h → paddle/fluid/eager/api/utils/global_utils.h

@@ -11,27 +11,15 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+//
 
 #pragma once
 
 #include "paddle/fluid/eager/eager_tensor.h"
 #include "paddle/fluid/platform/enforce.h"
-#include "paddle/pten/api/all.h"
-#include "paddle/pten/include/core.h"
+
 namespace egr {
 
-// Public
-void ScaleAPI(const egr::EagerTensor& x, float scale, float bias,
-              bool bias_after_scale, egr::EagerTensor* out);
-void FillConstAPI(double value, const pten::DDim& ddim,
-                  const paddle::platform::Place& place,
-                  const pten::DataType& dtype, const pten::DataLayout& layout,
-                  egr::EagerTensor* target);
-void FillConstAPI(double value, const paddle::framework::DDim& ddim,
-                  const paddle::platform::Place& place,
-                  const paddle::framework::proto::VarType::Type& dtype,
-                  egr::EagerTensor* target);
-//
 class UniqueNameGenerator {
  public:
   explicit UniqueNameGenerator(std::string prefix = "") : prefix_(prefix) {}