Support BF16 training for sharding and dp

paddle/fluid/distributed/collective/reducer.cc

@@ -254,6 +254,10 @@ static void ConcatTensorsWithType(
       ConcatTensorsForAllReduce<DeviceContext, double>()(
           context, dense_tensors_, p_dense_contents);
       break;
+    case phi::DataType::BFLOAT16:
+      ConcatTensorsForAllReduce<DeviceContext, platform::bfloat16>()(
+          context, dense_tensors_, p_dense_contents);
+      break;
     default:
       PADDLE_THROW(platform::errors::Unimplemented(
           "Data type (%s) is not supported when it concats tensors for "
@@ -281,6 +285,10 @@ static void SplitTensorsWithType(const DeviceContext &context,
       SplitTensorsForAllReduce<DeviceContext, double>()(
           context, p_dense_contents, p_dense_tensors);
       break;
+    case phi::DataType::BFLOAT16:
+      SplitTensorsForAllReduce<DeviceContext, platform::bfloat16>()(
+          context, p_dense_contents, p_dense_tensors);
+      break;
     default:
       PADDLE_THROW(platform::errors::Unimplemented(
           "Data type (%s) is not supported when it splits tensors for "

paddle/phi/kernels/cpu/fill_grad_kernel.cc

@@ -26,4 +26,5 @@ PD_REGISTER_KERNEL(fill_grad,
                    int64_t,
                    int,
                    paddle::platform::float16,
+                   paddle::platform::bfloat16,
                    bool) {}

paddle/phi/kernels/cpu/fill_kernel.cc

@@ -26,4 +26,5 @@ PD_REGISTER_KERNEL(fill,
                    int64_t,
                    int,
                    paddle::platform::float16,
+                   paddle::platform::bfloat16,
                    bool) {}

paddle/phi/kernels/gpu/fill_grad_kernel.cu

@@ -27,4 +27,5 @@ PD_REGISTER_KERNEL(fill_grad,
                    int64_t,
                    int,
                    paddle::platform::float16,
+                   paddle::platform::bfloat16,
                    bool) {}

paddle/phi/kernels/gpu/fill_kernel.cu

@@ -27,4 +27,5 @@ PD_REGISTER_KERNEL(fill,
                    int64_t,
                    int,
                    paddle::platform::float16,
+                   paddle::platform::bfloat16,
                    bool) {}

python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py

@@ -43,6 +43,7 @@
 alignment = {"gpu": 256, "cpu": 4096}
 align = {
     Type.fp16.value: 2,
+    Type.bf16.value: 2,
     Type.fp32.value: 4,
 }
 

python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py

@@ -531,6 +531,12 @@ def _rank_buffer_size(self, buffer_max_size, model_size):
                 "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======"
                 .format(rank_buffer_size[Type.fp16.value] / 2**19,
                         model_size / 2**19))
+        if Type.bf16.value in rank_buffer_size.keys():
+            # FP16 GradStorage and model size
+            logger_.info(
+                "====== BF16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======"
+                .format(rank_buffer_size[Type.bf16.value] / 2**19,
+                        model_size / 2**19))
         if Type.fp32.value in rank_buffer_size.keys():
             # FP32 GradStorage and model size
             logger_.info(

python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py

@@ -53,6 +53,8 @@ def __init__(self, size, dtype, device, convert_cpu=False):
                 dtype=np.float16) if Type.fp16.value == dtype else np.zeros(
                     size, dtype=np.float32)
             self.buffer = core.eager.Tensor(value=value, place=core.CPUPlace())
+            if dtype == Type.bf16.value:
+                self.buffer = paddle.cast(self.buffer, dtype=paddle.bfloat16)
         else:
             self.buffer = paddle.zeros(size, dtype=dtype)
 

python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py

@@ -41,6 +41,7 @@ class Type(Enum):
     Type of trainable parameters
     """
     fp16 = paddle.float16
+    bf16 = paddle.bfloat16
     fp32 = paddle.float32
 
 

python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py

@@ -45,6 +45,7 @@ class Type(Enum):
     Type of trainable parameters
     """
     fp16 = paddle.float16
+    bf16 = paddle.bfloat16
     fp32 = paddle.float32