Revert "Support BF16 training for sharding (PaddlePaddle#46846)"

This reverts commit 6adbed6.
GhostScreaming · Oct 21, 2022 · d58c1f6 · d58c1f6
1 parent 4345cb2
commit d58c1f6
Show file tree

Hide file tree

Showing 10 changed files with 0 additions and 23 deletions.
diff --git a/paddle/fluid/distributed/collective/reducer.cc b/paddle/fluid/distributed/collective/reducer.cc
@@ -254,10 +254,6 @@ static void ConcatTensorsWithType(
       ConcatTensorsForAllReduce<DeviceContext, double>()(
           context, dense_tensors_, p_dense_contents);
       break;
-    case phi::DataType::BFLOAT16:
-      ConcatTensorsForAllReduce<DeviceContext, platform::bfloat16>()(
-          context, dense_tensors_, p_dense_contents);
-      break;
     default:
       PADDLE_THROW(platform::errors::Unimplemented(
           "Data type (%s) is not supported when it concats tensors for "
@@ -285,10 +281,6 @@ static void SplitTensorsWithType(const DeviceContext &context,
       SplitTensorsForAllReduce<DeviceContext, double>()(
           context, p_dense_contents, p_dense_tensors);
       break;
-    case phi::DataType::BFLOAT16:
-      SplitTensorsForAllReduce<DeviceContext, platform::bfloat16>()(
-          context, p_dense_contents, p_dense_tensors);
-      break;
     default:
       PADDLE_THROW(platform::errors::Unimplemented(
           "Data type (%s) is not supported when it splits tensors for "

diff --git a/paddle/phi/kernels/cpu/fill_grad_kernel.cc b/paddle/phi/kernels/cpu/fill_grad_kernel.cc
@@ -26,5 +26,4 @@ PD_REGISTER_KERNEL(fill_grad,
                    int64_t,
                    int,
                    paddle::platform::float16,
-                   paddle::platform::bfloat16,
                    bool) {}
diff --git a/paddle/phi/kernels/cpu/fill_kernel.cc b/paddle/phi/kernels/cpu/fill_kernel.cc
@@ -26,5 +26,4 @@ PD_REGISTER_KERNEL(fill,
                    int64_t,
                    int,
                    paddle::platform::float16,
-                   paddle::platform::bfloat16,
                    bool) {}
diff --git a/paddle/phi/kernels/gpu/fill_grad_kernel.cu b/paddle/phi/kernels/gpu/fill_grad_kernel.cu
@@ -27,5 +27,4 @@ PD_REGISTER_KERNEL(fill_grad,
                    int64_t,
                    int,
                    paddle::platform::float16,
-                   paddle::platform::bfloat16,
                    bool) {}
diff --git a/paddle/phi/kernels/gpu/fill_kernel.cu b/paddle/phi/kernels/gpu/fill_kernel.cu
@@ -27,5 +27,4 @@ PD_REGISTER_KERNEL(fill,
                    int64_t,
                    int,
                    paddle::platform::float16,
-                   paddle::platform::bfloat16,
                    bool) {}
diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py
@@ -43,7 +43,6 @@
 alignment = {"gpu": 256, "cpu": 4096}
 align = {
     Type.fp16.value: 2,
-    Type.bf16.value: 2,
     Type.fp32.value: 4,
 }
 

diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py
@@ -531,12 +531,6 @@ def _rank_buffer_size(self, buffer_max_size, model_size):
                 "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======"
                 .format(rank_buffer_size[Type.fp16.value] / 2**19,
                         model_size / 2**19))
-        if Type.bf16.value in rank_buffer_size.keys():
-            # FP16 GradStorage and model size
-            logger_.info(
-                "====== BF16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======"
-                .format(rank_buffer_size[Type.bf16.value] / 2**19,
-                        model_size / 2**19))
         if Type.fp32.value in rank_buffer_size.keys():
             # FP32 GradStorage and model size
             logger_.info(

diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py
@@ -53,8 +53,6 @@ def __init__(self, size, dtype, device, convert_cpu=False):
                 dtype=np.float16) if Type.fp16.value == dtype else np.zeros(
                     size, dtype=np.float32)
             self.buffer = core.eager.Tensor(value=value, place=core.CPUPlace())
-            if dtype == Type.bf16.value:
-                self.buffer = paddle.cast(self.buffer, dtype=paddle.bfloat16)
         else:
             self.buffer = paddle.zeros(size, dtype=dtype)
 

diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py
@@ -41,7 +41,6 @@ class Type(Enum):
     Type of trainable parameters
     """
     fp16 = paddle.float16
-    bf16 = paddle.bfloat16
     fp32 = paddle.float32
 
 

diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py
@@ -45,7 +45,6 @@ class Type(Enum):
     Type of trainable parameters
     """
     fp16 = paddle.float16
-    bf16 = paddle.bfloat16
     fp32 = paddle.float32