[hybrid npu] fix npu found_finite in hybrid (PaddlePaddle#35134) (Pad…

…dlePaddle#35291) Co-authored-by: WangXi <wangxi16@baidu.com>
FeixLiu · Aug 31, 2021 · e64105f · e64105f
1 parent 6fb58ae
commit e64105f
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
@@ -369,8 +369,11 @@ def _adapt_amp_clip_without_sharding(self):
         # FIXME(wangxi): mp should prune duplicated param_grads when calc
         # amp inf_var & clip global_norm_var
 
-        FP16Utils.sync_amp_check_nan_inf(main_block,
-                                         [self.mp_ring_id, self.pp_ring_id])
+        rings = [self.mp_ring_id, self.pp_ring_id]
+        # FIXME(wangxi): some problem with NPU found_finite, need sync with DP
+        if core.is_compiled_with_npu():
+            rings += [self.dp_ring_id]
+        FP16Utils.sync_amp_check_nan_inf(main_block, rings)
 
         gradientclip_helper = GradientClipHelper(None)
         gradientclip_helper.sync_global_norm(