From 54ab656c7b654de96f345f03f2887b7f4616d456 Mon Sep 17 00:00:00 2001 From: Zhong Hui Date: Tue, 27 Apr 2021 19:51:32 +0800 Subject: [PATCH 001/156] [OPs] Bug fix, fix the segment mean for illegal syncthreads usage. (#32596) (#32610) * [OPs] Bug fix, fix the segment mean for illegal syncthreads usage. --- .../fluid/operators/math/segment_pooling.cu | 116 ++++++++++++------ 1 file changed, 78 insertions(+), 38 deletions(-) diff --git a/paddle/fluid/operators/math/segment_pooling.cu b/paddle/fluid/operators/math/segment_pooling.cu index 0b615cefac4ee..b49b5036ac42e 100644 --- a/paddle/fluid/operators/math/segment_pooling.cu +++ b/paddle/fluid/operators/math/segment_pooling.cu @@ -25,14 +25,12 @@ namespace operators { using Tensor = framework::Tensor; template -__global__ void SegmentMeanCustomKernel( - const Index* segment_ids, const T* input, T* output, T* summed_ids, - const Index input_length_size, const Index inner_dim_size, - const Index output_length_size, const Index total_stripe_count) { +__global__ void SegmentSumIdsKernel(const Index* segment_ids, T* summed_ids, + const Index input_length_size, + const Index total_stripe_count) { CUDA_KERNEL_LOOP(stripe_index, total_stripe_count) { - const Index segment_offset = stripe_index % inner_dim_size; - const Index dim_index_base = - stripe_index / inner_dim_size * Index(DimTileSize); + const Index segment_offset = stripe_index; + const Index dim_index_base = stripe_index * Index(DimTileSize); const Index actual_height = min(Index(DimTileSize), input_length_size - dim_index_base); @@ -41,19 +39,20 @@ __global__ void SegmentMeanCustomKernel( if (dim_index_base > 0) { last_segment_id = segment_ids[dim_index_base - 1]; } - if (segment_offset == 0) { - T sum = T(0); - for (Index j = 0; j < actual_height; j++) { - Index current_segment_id = segment_ids[dim_index_base + j]; - // Note(ZHUI): following check may cause - // cudaErrorLaunchOutOfResources. - // PADDLE_ENFORCE(current_segment_id >= last_segment_id, - // "the segment ids should be sorted, but got " - // "segment_ids[%d]:%d > segment_ids[%d]:%d.", - // dim_index_base + j - 1, dim_index_base + j, - // last_segment_id, current_segment_id); - - if (j > 0 && current_segment_id > last_segment_id) { + T sum = T(0); + for (Index j = 0; j < actual_height; j++) { + Index current_segment_id = segment_ids[dim_index_base + j]; + PADDLE_ENFORCE(current_segment_id >= last_segment_id, + "the segment ids should be sorted, but got " + "segment_ids[%d]:%d > segment_ids[%d]:%d.", + dim_index_base + j - 1, dim_index_base + j, + last_segment_id, current_segment_id); + if (current_segment_id > last_segment_id) { + for (Index interval_id = last_segment_id + 1; + interval_id < current_segment_id; ++interval_id) { + *(summed_ids + interval_id) = 0; + } + if (j > 0) { if (last_segment_id == first_segment_id) { platform::CudaAtomicAdd(summed_ids + last_segment_id, sum); } else { @@ -61,33 +60,60 @@ __global__ void SegmentMeanCustomKernel( } sum = T(0); } - sum += T(1); - last_segment_id = current_segment_id; } - platform::CudaAtomicAdd(summed_ids + last_segment_id, sum); + sum += T(1); + last_segment_id = current_segment_id; + } + platform::CudaAtomicAdd(summed_ids + last_segment_id, sum); + } +} + +template +__global__ void SegmentMeanKernel(const Index* segment_ids, const T* input, + T* output, T* summed_ids, + const Index input_length_size, + const Index inner_dim_size, + const Index output_length_size, + const Index total_stripe_count) { + CUDA_KERNEL_LOOP(stripe_index, total_stripe_count) { + const Index segment_offset = stripe_index % inner_dim_size; + const Index dim_index_base = + stripe_index / inner_dim_size * Index(DimTileSize); + const Index actual_height = + min(Index(DimTileSize), input_length_size - dim_index_base); + + Index first_segment_id = segment_ids[dim_index_base]; + Index last_segment_id = -1; + if (dim_index_base > 0) { + last_segment_id = segment_ids[dim_index_base - 1]; } - // ensure last_segment_id is the largest - last_segment_id = output_length_size; - __syncthreads(); T sum = T(0); for (Index j = 0; j < actual_height; j++) { Index current_segment_id = segment_ids[dim_index_base + j]; if (current_segment_id > last_segment_id) { - const Index output_index = - last_segment_id * inner_dim_size + segment_offset; - if (last_segment_id == first_segment_id) { - platform::CudaAtomicAdd(output + output_index, - sum / *(summed_ids + last_segment_id)); - } else { - *(output + output_index) = sum / *(summed_ids + last_segment_id); + // reset the interval value which do not have corresponding ids. + for (Index interval_id = last_segment_id + 1; + interval_id < current_segment_id; ++interval_id) { + *(output + interval_id * inner_dim_size + segment_offset) = T(0); + } + + if (j > 0) { + Index output_index = + last_segment_id * inner_dim_size + segment_offset; + + if (last_segment_id == first_segment_id) { + platform::CudaAtomicAdd(output + output_index, + sum / *(summed_ids + last_segment_id)); + } else { + *(output + output_index) = sum / *(summed_ids + last_segment_id); + } + sum = T(0); } - sum = T(0); } sum += input[(dim_index_base + j) * inner_dim_size + segment_offset]; last_segment_id = current_segment_id; } - const Index output_index = - last_segment_id * inner_dim_size + segment_offset; + Index output_index = last_segment_id * inner_dim_size + segment_offset; platform::CudaAtomicAdd(output + output_index, sum / *(summed_ids + last_segment_id)); } @@ -122,7 +148,7 @@ __global__ void SegmentOpsKernel(const Index* segment_ids, const T* input, // reset the interval value which do not have corresponding ids. for (Index interval_id = last_segment_id + 1; interval_id < current_segment_id; ++interval_id) { - *(output + interval_id * inner_dim_size + segment_offset) = 0; + *(output + interval_id * inner_dim_size + segment_offset) = T(0); } // don't update result when j=0 if (j > 0) { @@ -272,11 +298,25 @@ class SegmentPoolFunctor { framework::Tensor* output, framework::Tensor* summed_ids = nullptr, const std::string pooltype = "SUM") { + if (pooltype == "MEAN") { + // Sum the segment id num first + T DimTileSize = 8; + auto input_length_size = segment_ids.numel(); + auto total_stripe_count = + (input_length_size + DimTileSize - 1) / DimTileSize; + auto config = platform::GetGpuLaunchConfig1D(ctx, total_stripe_count); + SegmentSumIdsKernel< + T, IndexT, IndexT(8)><<>>( + segment_ids.data(), summed_ids->data(), input_length_size, + total_stripe_count); + } + auto h = ArrangeHelper(input.numel(), segment_ids.dims()[0], output->dims()[0]); auto config = platform::GetGpuLaunchConfig1D(ctx, h.total_stripe_count); if (pooltype == "MEAN") { - SegmentMeanCustomKernel< + SegmentMeanKernel< T, IndexT, IndexT(8)><<>>( segment_ids.data(), input.data(), output->data(), From 938a5a53d673f0af2a604314f81fde239f38d7ca Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Wed, 28 Apr 2021 10:31:16 +0800 Subject: [PATCH 002/156] cherry-pick from develop: update 2.0 public api in nn #31912 (#32621) * update 2.0 public api in nn * replace Chinese character cause error in ci;synchronization with pr:#32588 to avoid 'ascii' codec in python2 * numbers used in paddle.nn.functional.norm but not imported --- .../fleet/parameter_server/ir/trainer_pass.py | 2 +- .../fluid/tests/unittests/hccl_tools.py | 2 +- python/paddle/nn/__init__.py | 415 ++++++++++++------ python/paddle/nn/clip.py | 8 +- python/paddle/nn/decode.py | 9 +- python/paddle/nn/functional/__init__.py | 382 ++++++++-------- python/paddle/nn/functional/activation.py | 45 +- python/paddle/nn/functional/common.py | 30 +- python/paddle/nn/functional/conv.py | 9 - python/paddle/nn/functional/extension.py | 2 - python/paddle/nn/functional/input.py | 2 - python/paddle/nn/functional/loss.py | 35 +- python/paddle/nn/functional/norm.py | 11 - python/paddle/nn/functional/pooling.py | 15 - python/paddle/nn/functional/vision.py | 37 -- python/paddle/nn/initializer/__init__.py | 50 +-- python/paddle/nn/initializer/assign.py | 2 - python/paddle/nn/initializer/constant.py | 2 - python/paddle/nn/initializer/kaiming.py | 2 - python/paddle/nn/initializer/normal.py | 2 - python/paddle/nn/initializer/uniform.py | 2 - python/paddle/nn/initializer/xavier.py | 2 - python/paddle/nn/layer/__init__.py | 150 +++---- python/paddle/nn/layer/activation.py | 27 -- python/paddle/nn/layer/common.py | 20 +- python/paddle/nn/layer/conv.py | 9 - python/paddle/nn/layer/distance.py | 2 - python/paddle/nn/layer/loss.py | 18 +- python/paddle/nn/layer/norm.py | 13 +- python/paddle/nn/layer/pooling.py | 15 - python/paddle/nn/layer/rnn.py | 12 - python/paddle/nn/layer/transformer.py | 8 - python/paddle/nn/layer/vision.py | 2 - python/paddle/nn/utils/__init__.py | 7 +- python/paddle/nn/utils/weight_norm_hook.py | 2 - python/paddle/utils/deprecated.py | 5 +- 36 files changed, 570 insertions(+), 786 deletions(-) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py index 5f32749704747..d4af3e2f8042a 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py @@ -527,7 +527,7 @@ def create_heter_program(program, config, heter_program, heter_ops, # This function mainly includes the following contents: # 1. For every heter block: # a) copy heter device op from origin program - # b) create variables which belong to heter op： + # b) create variables which belong to heter op: # -> if variable is persistable, clone it in global_scope # -> if variable is temp, create it in heter block # c) create communicate related op as follow: diff --git a/python/paddle/fluid/tests/unittests/hccl_tools.py b/python/paddle/fluid/tests/unittests/hccl_tools.py index 3ae8f38dc64bd..e3628ee5a4e9b 100644 --- a/python/paddle/fluid/tests/unittests/hccl_tools.py +++ b/python/paddle/fluid/tests/unittests/hccl_tools.py @@ -58,7 +58,7 @@ def parse_args(): default="[0,8)", help="The number of the Ascend accelerators used. please note that the Ascend accelerators" "used must be continuous, such [0,4) means to use four chips " - "0，1，2，3; [0,1) means to use chip 0; The first four chips are" + "0,1,2,3; [0,1) means to use chip 0; The first four chips are" "a group, and the last four chips are a group. In addition to" "the [0,8) chips are allowed, other cross-group such as [3,6)" "are prohibited.") diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 836d4008f7d0b..d2f0063af0d22 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -15,148 +15,273 @@ # TODO: import all neural network related api under this directory, # including layers, linear, conv, rnn etc. -from .layer import norm -from .functional import extension -from .layer import common -from .layer import rnn -from .utils import weight_norm_hook - -from . import initializer - -__all__ = [] -__all__ += norm.__all__ -__all__ += extension.__all__ -__all__ += common.__all__ -__all__ += rnn.__all__ -__all__ += weight_norm_hook.__all__ - -# TODO: define alias in nn directory -from .clip import ClipGradByGlobalNorm #DEFINE_ALIAS -from .clip import ClipGradByNorm #DEFINE_ALIAS -from .clip import ClipGradByValue #DEFINE_ALIAS -# from .control_flow import cond #DEFINE_ALIAS -# from .control_flow import DynamicRNN #DEFINE_ALIAS -# from .control_flow import StaticRNN #DEFINE_ALIAS -# from .control_flow import while_loop #DEFINE_ALIAS -# from .control_flow import rnn #DEFINE_ALIAS -from .decode import BeamSearchDecoder #DEFINE_ALIAS -from .decode import dynamic_decode #DEFINE_ALIAS -# from .decode import Decoder #DEFINE_ALIAS -# from .decode import crf_decoding #DEFINE_ALIAS -# from .decode import ctc_greedy_decoder #DEFINE_ALIAS -# from .input import Input #DEFINE_ALIAS -from .layer.activation import ELU #DEFINE_ALIAS -from .layer.activation import GELU #DEFINE_ALIAS -from .layer.activation import Tanh #DEFINE_ALIAS -from .layer.activation import Hardshrink #DEFINE_ALIAS -from .layer.activation import Hardswish #DEFINE_ALIAS -from .layer.activation import Hardtanh #DEFINE_ALIAS -from .layer.activation import PReLU #DEFINE_ALIAS -from .layer.activation import ReLU #DEFINE_ALIAS -from .layer.activation import ReLU6 #DEFINE_ALIAS -from .layer.activation import SELU #DEFINE_ALIAS -from .layer.activation import Silu #DEFINE_ALIAS -from .layer.activation import LeakyReLU #DEFINE_ALIAS -from .layer.activation import Sigmoid #DEFINE_ALIAS -from .layer.activation import Hardsigmoid #DEFINE_ALIAS -from .layer.activation import LogSigmoid #DEFINE_ALIAS -from .layer.activation import Softmax #DEFINE_ALIAS -from .layer.activation import Softplus #DEFINE_ALIAS -from .layer.activation import Softshrink #DEFINE_ALIAS -from .layer.activation import Softsign #DEFINE_ALIAS -from .layer.activation import Swish #DEFINE_ALIAS -from .layer.activation import Tanhshrink #DEFINE_ALIAS -from .layer.activation import ThresholdedReLU #DEFINE_ALIAS -from .layer.activation import LogSoftmax #DEFINE_ALIAS -from .layer.activation import Maxout #DEFINE_ALIAS -from .layer.common import Pad1D #DEFINE_ALIAS -from .layer.common import Pad2D #DEFINE_ALIAS -from .layer.common import Pad3D #DEFINE_ALIAS -from .layer.common import CosineSimilarity #DEFINE_ALIAS -from .layer.common import Embedding #DEFINE_ALIAS -from .layer.common import Linear #DEFINE_ALIAS -from .layer.common import Flatten #DEFINE_ALIAS -from .layer.common import Upsample #DEFINE_ALIAS -from .layer.common import UpsamplingNearest2D #DEFINE_ALIAS -from .layer.common import UpsamplingBilinear2D #DEFINE_ALIAS -from .layer.common import Bilinear #DEFINE_ALIAS -from .layer.common import Dropout #DEFINE_ALIAS -from .layer.common import Dropout2D #DEFINE_ALIAS -from .layer.common import Dropout3D #DEFINE_ALIAS -from .layer.common import AlphaDropout #DEFINE_ALIAS -from .layer.common import Unfold #DEFINE_ALIAS - -from .layer.pooling import AvgPool1D #DEFINE_ALIAS -from .layer.pooling import AvgPool2D #DEFINE_ALIAS -from .layer.pooling import AvgPool3D #DEFINE_ALIAS -from .layer.pooling import MaxPool1D #DEFINE_ALIAS -from .layer.pooling import MaxPool2D #DEFINE_ALIAS -from .layer.pooling import MaxPool3D #DEFINE_ALIAS -from .layer.pooling import AdaptiveAvgPool1D #DEFINE_ALIAS -from .layer.pooling import AdaptiveAvgPool2D #DEFINE_ALIAS -from .layer.pooling import AdaptiveAvgPool3D #DEFINE_ALIAS - -from .layer.pooling import AdaptiveMaxPool1D #DEFINE_ALIAS -from .layer.pooling import AdaptiveMaxPool2D #DEFINE_ALIAS -from .layer.pooling import AdaptiveMaxPool3D #DEFINE_ALIAS -from .layer.conv import Conv1D #DEFINE_ALIAS -from .layer.conv import Conv2D #DEFINE_ALIAS -from .layer.conv import Conv3D #DEFINE_ALIAS -from .layer.conv import Conv1DTranspose #DEFINE_ALIAS -from .layer.conv import Conv2DTranspose #DEFINE_ALIAS -from .layer.conv import Conv3DTranspose #DEFINE_ALIAS -# from .layer.conv import TreeConv #DEFINE_ALIAS -# from .layer.conv import Conv1D #DEFINE_ALIAS -from .layer.common import Linear -# from .layer.loss import NCELoss #DEFINE_ALIAS -from .layer.loss import BCEWithLogitsLoss #DEFINE_ALIAS -from .layer.loss import CrossEntropyLoss #DEFINE_ALIAS -from .layer.loss import HSigmoidLoss #DEFINE_ALIAS -from .layer.loss import MSELoss #DEFINE_ALIAS -from .layer.loss import L1Loss #DEFINE_ALIAS -from .layer.loss import NLLLoss #DEFINE_ALIAS -from .layer.loss import BCELoss #DEFINE_ALIAS -from .layer.loss import KLDivLoss #DEFINE_ALIAS -from .layer.loss import MarginRankingLoss #DEFINE_ALIAS -from .layer.loss import CTCLoss #DEFINE_ALIAS -from .layer.loss import SmoothL1Loss #DEFINE_ALIAS -from .layer.norm import BatchNorm #DEFINE_ALIAS -from .layer.norm import SyncBatchNorm #DEFINE_ALIAS -from .layer.norm import GroupNorm #DEFINE_ALIAS -from .layer.norm import LayerNorm #DEFINE_ALIAS -from .layer.norm import SpectralNorm #DEFINE_ALIAS -from .layer.norm import InstanceNorm1D #DEFINE_ALIAS -from .layer.norm import InstanceNorm2D #DEFINE_ALIAS -from .layer.norm import InstanceNorm3D #DEFINE_ALIAS -from .layer.norm import BatchNorm1D #DEFINE_ALIAS -from .layer.norm import BatchNorm2D #DEFINE_ALIAS -from .layer.norm import BatchNorm3D #DEFINE_ALIAS -from .layer.norm import LocalResponseNorm #DEFINE_ALIAS - -from .layer.rnn import RNNCellBase #DEFINE_ALIAS -from .layer.rnn import SimpleRNNCell #DEFINE_ALIAS -from .layer.rnn import LSTMCell #DEFINE_ALIAS -from .layer.rnn import GRUCell #DEFINE_ALIAS -from .layer.rnn import RNN #DEFINE_ALIAS -from .layer.rnn import BiRNN #DEFINE_ALIAS -from .layer.rnn import SimpleRNN #DEFINE_ALIAS -from .layer.rnn import LSTM #DEFINE_ALIAS -from .layer.rnn import GRU #DEFINE_ALIAS - -from .layer.transformer import MultiHeadAttention -from .layer.transformer import TransformerEncoderLayer -from .layer.transformer import TransformerEncoder -from .layer.transformer import TransformerDecoderLayer -from .layer.transformer import TransformerDecoder -from .layer.transformer import Transformer -from .layer.distance import PairwiseDistance #DEFINE_ALIAS - -from .layer.vision import PixelShuffle - -from .layer.container import LayerDict #DEFINE_ALIAS - -from .layer import loss #DEFINE_ALIAS -from .layer import conv #DEFINE_ALIAS -from .layer import vision #DEFINE_ALIAS -from ..fluid.dygraph.layers import Layer #DEFINE_ALIAS -from ..fluid.dygraph.container import LayerList, ParameterList, Sequential #DEFINE_ALIAS +from .clip import ClipGradByGlobalNorm # noqa: F401 +from .clip import ClipGradByNorm # noqa: F401 +from .clip import ClipGradByValue # noqa: F401 +from .decode import BeamSearchDecoder # noqa: F401 +from .decode import dynamic_decode # noqa: F401 +from .layer.activation import ELU # noqa: F401 +from .layer.activation import GELU # noqa: F401 +from .layer.activation import Tanh # noqa: F401 +from .layer.activation import Hardshrink # noqa: F401 +from .layer.activation import Hardswish # noqa: F401 +from .layer.activation import Hardtanh # noqa: F401 +from .layer.activation import PReLU # noqa: F401 +from .layer.activation import ReLU # noqa: F401 +from .layer.activation import ReLU6 # noqa: F401 +from .layer.activation import SELU # noqa: F401 +from .layer.activation import Silu # noqa: F401 +from .layer.activation import LeakyReLU # noqa: F401 +from .layer.activation import Sigmoid # noqa: F401 +from .layer.activation import Hardsigmoid # noqa: F401 +from .layer.activation import LogSigmoid # noqa: F401 +from .layer.activation import Softmax # noqa: F401 +from .layer.activation import Softplus # noqa: F401 +from .layer.activation import Softshrink # noqa: F401 +from .layer.activation import Softsign # noqa: F401 +from .layer.activation import Swish # noqa: F401 +from .layer.activation import Tanhshrink # noqa: F401 +from .layer.activation import ThresholdedReLU # noqa: F401 +from .layer.activation import LogSoftmax # noqa: F401 +from .layer.activation import Maxout # noqa: F401 +from .layer.common import Pad1D # noqa: F401 +from .layer.common import Pad2D # noqa: F401 +from .layer.common import Pad3D # noqa: F401 +from .layer.common import CosineSimilarity # noqa: F401 +from .layer.common import Embedding # noqa: F401 +from .layer.common import Linear # noqa: F401 +from .layer.common import Flatten # noqa: F401 +from .layer.common import Upsample # noqa: F401 +from .layer.common import UpsamplingNearest2D # noqa: F401 +from .layer.common import UpsamplingBilinear2D # noqa: F401 +from .layer.common import Bilinear # noqa: F401 +from .layer.common import Dropout # noqa: F401 +from .layer.common import Dropout2D # noqa: F401 +from .layer.common import Dropout3D # noqa: F401 +from .layer.common import AlphaDropout # noqa: F401 +from .layer.common import Unfold # noqa: F401 + +from .layer.pooling import AvgPool1D # noqa: F401 +from .layer.pooling import AvgPool2D # noqa: F401 +from .layer.pooling import AvgPool3D # noqa: F401 +from .layer.pooling import MaxPool1D # noqa: F401 +from .layer.pooling import MaxPool2D # noqa: F401 +from .layer.pooling import MaxPool3D # noqa: F401 +from .layer.pooling import AdaptiveAvgPool1D # noqa: F401 +from .layer.pooling import AdaptiveAvgPool2D # noqa: F401 +from .layer.pooling import AdaptiveAvgPool3D # noqa: F401 +from .layer.pooling import AdaptiveMaxPool1D # noqa: F401 +from .layer.pooling import AdaptiveMaxPool2D # noqa: F401 +from .layer.pooling import AdaptiveMaxPool3D # noqa: F401 + +from .layer.conv import Conv1D # noqa: F401 +from .layer.conv import Conv2D # noqa: F401 +from .layer.conv import Conv3D # noqa: F401 +from .layer.conv import Conv1DTranspose # noqa: F401 +from .layer.conv import Conv2DTranspose # noqa: F401 +from .layer.conv import Conv3DTranspose # noqa: F401 + +from .layer.loss import BCEWithLogitsLoss # noqa: F401 +from .layer.loss import CrossEntropyLoss # noqa: F401 +from .layer.loss import HSigmoidLoss # noqa: F401 +from .layer.loss import MSELoss # noqa: F401 +from .layer.loss import L1Loss # noqa: F401 +from .layer.loss import NLLLoss # noqa: F401 +from .layer.loss import BCELoss # noqa: F401 +from .layer.loss import KLDivLoss # noqa: F401 +from .layer.loss import MarginRankingLoss # noqa: F401 +from .layer.loss import CTCLoss # noqa: F401 +from .layer.loss import SmoothL1Loss # noqa: F401 +from .layer.norm import BatchNorm # noqa: F401 +from .layer.norm import SyncBatchNorm # noqa: F401 +from .layer.norm import GroupNorm # noqa: F401 +from .layer.norm import LayerNorm # noqa: F401 +from .layer.norm import SpectralNorm # noqa: F401 +from .layer.norm import InstanceNorm1D # noqa: F401 +from .layer.norm import InstanceNorm2D # noqa: F401 +from .layer.norm import InstanceNorm3D # noqa: F401 +from .layer.norm import BatchNorm1D # noqa: F401 +from .layer.norm import BatchNorm2D # noqa: F401 +from .layer.norm import BatchNorm3D # noqa: F401 +from .layer.norm import LocalResponseNorm # noqa: F401 + +from .layer.rnn import RNNCellBase # noqa: F401 +from .layer.rnn import SimpleRNNCell # noqa: F401 +from .layer.rnn import LSTMCell # noqa: F401 +from .layer.rnn import GRUCell # noqa: F401 +from .layer.rnn import RNN # noqa: F401 +from .layer.rnn import BiRNN # noqa: F401 +from .layer.rnn import SimpleRNN # noqa: F401 +from .layer.rnn import LSTM # noqa: F401 +from .layer.rnn import GRU # noqa: F401 + +from .layer.transformer import MultiHeadAttention # noqa: F401 +from .layer.transformer import TransformerEncoderLayer # noqa: F401 +from .layer.transformer import TransformerEncoder # noqa: F401 +from .layer.transformer import TransformerDecoderLayer # noqa: F401 +from .layer.transformer import TransformerDecoder # noqa: F401 +from .layer.transformer import Transformer # noqa: F401 +from .layer.distance import PairwiseDistance # noqa: F401 + +from .layer.vision import PixelShuffle # noqa: F401 +from .layer.container import LayerDict # noqa: F401 + +# TODO: remove loss, keep it for too many used in unitests +from .layer import loss # noqa: F401 +from ..fluid.dygraph.layers import Layer # noqa: F401 +from ..fluid.dygraph.container import LayerList # noqa: F401 +from ..fluid.dygraph.container import ParameterList # noqa: F401 +from ..fluid.dygraph.container import Sequential # noqa: F401 + +from . import utils # noqa: F401 +from . import functional # noqa: F401 +from . import initializer # noqa: F401 + +#TODO: remove 'diag_embed', 'remove_weight_norm', 'weight_norm' months later. +import paddle.utils.deprecated as deprecated + + +@deprecated( + since="2.0.0", + update_to="paddle.nn.funcitional.diag_embed", + reason="diag_embed in paddle.nn will removed in future") +def diag_embed(*args): + ''' + alias name of paddle.nn.functional.diag_embed + ''' + return functional.diag_embed(*args) + + +@deprecated( + since="2.0.0", + update_to="paddle.nn.utils.remove_weight_norm", + reason="remove_weight_norm in paddle.nn will removed in future") +def remove_weight_norm(*args): + ''' + alias name of paddle.nn.utils.remove_weight_norm + ''' + return utils.remove_weight_norm(*args) + + +@deprecated( + since="2.0.0", + update_to="paddle.nn.utils.weight_norm", + reason="weight_norm in paddle.nn will removed in future") +def weight_norm(*args): + ''' + alias name of paddle.nn.utils.weight_norm + ''' + return utils.weight_norm(*args) + + +__all__ = [ #noqa + 'BatchNorm', + 'GroupNorm', + 'LayerNorm', + 'SpectralNorm', + 'BatchNorm1D', + 'BatchNorm2D', + 'BatchNorm3D', + 'InstanceNorm1D', + 'InstanceNorm2D', + 'InstanceNorm3D', + 'SyncBatchNorm', + 'LocalResponseNorm', + 'Embedding', + 'Linear', + 'Upsample', + 'UpsamplingNearest2D', + 'UpsamplingBilinear2D', + 'Pad1D', + 'Pad2D', + 'Pad3D', + 'CosineSimilarity', + 'Dropout', + 'Dropout2D', + 'Dropout3D', + 'Bilinear', + 'AlphaDropout', + 'Unfold' + 'RNNCellBase', + 'SimpleRNNCell', + 'LSTMCell', + 'GRUCell', + 'RNN', + 'BiRNN', + 'SimpleRNN', + 'LSTM', + 'GRU', + 'dynamic_decode', + 'MultiHeadAttention', + 'Maxout', + 'Softsign', + 'Transformer', + 'MSELoss', + 'LogSigmoid', + 'BeamSearchDecoder', + 'ClipGradByNorm', + 'ReLU', + 'PairwiseDistance', + 'BCEWithLogitsLoss', + 'SmoothL1Loss', + 'MaxPool3D', + 'AdaptiveMaxPool2D', + 'Hardshrink', + 'clip', + 'Softplus', + 'KLDivLoss', + 'clip_by_norm', + 'AvgPool2D', + 'L1Loss', + 'LeakyReLU', + 'AvgPool1D', + 'AdaptiveAvgPool3D', + 'AdaptiveMaxPool3D', + 'NLLLoss', + 'Conv1D', + 'Sequential', + 'Hardswish', + 'Conv1DTranspose', + 'AdaptiveMaxPool1D', + 'TransformerEncoder', + 'Softmax', + 'ParameterList', + 'Conv2D', + 'Softshrink', + 'Hardtanh', + 'TransformerDecoderLayer', + 'CrossEntropyLoss', + 'GELU', + 'SELU', + 'Silu', + 'Conv2DTranspose', + 'CTCLoss', + 'ThresholdedReLU', + 'AdaptiveAvgPool2D', + 'MaxPool1D', + 'Layer', + 'TransformerDecoder', + 'Conv3D', + 'Tanh', + 'Conv3DTranspose', + 'Flatten', + 'AdaptiveAvgPool1D', + 'Tanhshrink', + 'HSigmoidLoss', + 'PReLU', + 'TransformerEncoderLayer', + 'AvgPool3D', + 'MaxPool2D', + 'MarginRankingLoss', + 'LayerList', + 'ClipGradByValue', + 'BCELoss', + 'Hardsigmoid', + 'ClipGradByGlobalNorm', + 'LogSoftmax', + 'Sigmoid', + 'Swish', + 'PixelShuffle', + 'ELU', + 'ReLU6' +] diff --git a/python/paddle/nn/clip.py b/python/paddle/nn/clip.py index 9180a883e835c..70c49b4a53876 100644 --- a/python/paddle/nn/clip.py +++ b/python/paddle/nn/clip.py @@ -13,8 +13,6 @@ # limitations under the License. # TODO: define the functions to clip gradient of parameter -from ..fluid.clip import ClipGradByGlobalNorm #DEFINE_ALIAS -from ..fluid.clip import ClipGradByNorm #DEFINE_ALIAS -from ..fluid.clip import ClipGradByValue #DEFINE_ALIAS - -__all__ = ['ClipGradByGlobalNorm', 'ClipGradByNorm', 'ClipGradByValue'] +from ..fluid.clip import ClipGradByGlobalNorm # noqa: F401 +from ..fluid.clip import ClipGradByNorm # noqa: F401 +from ..fluid.clip import ClipGradByValue # noqa: F401 diff --git a/python/paddle/nn/decode.py b/python/paddle/nn/decode.py index bba5aba0da9ad..3229f0b21a669 100644 --- a/python/paddle/nn/decode.py +++ b/python/paddle/nn/decode.py @@ -12,10 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..fluid.layers import BeamSearchDecoder #DEFINE_ALIAS -from ..fluid.layers import dynamic_decode #DEFINE_ALIAS - -__all__ = [ - 'BeamSearchDecoder', - 'dynamic_decode', -] +from ..fluid.layers import BeamSearchDecoder # noqa: F401 +from ..fluid.layers import dynamic_decode # noqa: F401 diff --git a/python/paddle/nn/functional/__init__.py b/python/paddle/nn/functional/__init__.py index 98124be7288d0..d4c17a27a6178 100644 --- a/python/paddle/nn/functional/__init__.py +++ b/python/paddle/nn/functional/__init__.py @@ -14,211 +14,185 @@ # TODO: import all neural network related api under this directory, # including layers, linear, conv, rnn etc. -__all__ = [] -# TODO: define alias in functional directory -from . import conv -__all__ += conv.__all__ -from . import activation -__all__ += activation.__all__ -from . import extension -__all__ += extension.__all__ -from . import common -__all__ += common.__all__ -from . import pooling -__all__ += pooling.__all__ -from . import loss -__all__ += loss.__all__ -from .activation import elu #DEFINE_ALIAS -from .activation import elu_ #DEFINE_ALIAS -# from .activation import erf #DEFINE_ALIAS -from .activation import gelu #DEFINE_ALIAS -from .activation import hardshrink #DEFINE_ALIAS -from .activation import hardtanh #DEFINE_ALIAS -from .activation import hardsigmoid #DEFINE_ALIAS -from .activation import hardswish #DEFINE_ALIAS -from .activation import leaky_relu #DEFINE_ALIAS -from .activation import log_sigmoid #DEFINE_ALIAS -from .activation import maxout #DEFINE_ALIAS -from .activation import prelu #DEFINE_ALIAS -from .activation import relu #DEFINE_ALIAS -from .activation import relu_ #DEFINE_ALIAS -from .activation import relu6 #DEFINE_ALIAS -from .activation import selu #DEFINE_ALIAS -from .activation import sigmoid #DEFINE_ALIAS -from .activation import silu #DEFINE_ALIAS -# from .activation import soft_relu #DEFINE_ALIAS -from .activation import softmax #DEFINE_ALIAS -from .activation import softmax_ #DEFINE_ALIAS -from .activation import softplus #DEFINE_ALIAS -from .activation import softshrink #DEFINE_ALIAS -from .activation import softsign #DEFINE_ALIAS -from .activation import swish #DEFINE_ALIAS -from .activation import tanh #DEFINE_ALIAS -from .activation import tanh_ #DEFINE_ALIAS -from .activation import tanhshrink #DEFINE_ALIAS -from .activation import thresholded_relu #DEFINE_ALIAS -from .activation import log_softmax #DEFINE_ALIAS -from .activation import glu #DEFINE_ALIAS -from .common import dropout #DEFINE_ALIAS -from .common import dropout2d #DEFINE_ALIAS -from .common import dropout3d #DEFINE_ALIAS -from .common import alpha_dropout #DEFINE_ALIAS -# from .common import embedding #DEFINE_ALIAS -# from .common import fc #DEFINE_ALIAS -from .common import label_smooth -# from .common import one_hot #DEFINE_ALIAS -from .common import pad #DEFINE_ALIAS -# from .common import pad_constant_like #DEFINE_ALIAS -# from .common import pad2d #DEFINE_ALIAS -from .common import cosine_similarity #DEFINE_ALIAS -from .common import unfold #DEFINE_ALIAS -# from .common import bilinear_tensor_product #DEFINE_ALIAS -from .common import interpolate #DEFINE_ALIAS -from .common import upsample #DEFINE_ALIAS -from .common import bilinear #DEFINE_ALIAS -from .conv import conv1d #DEFINE_ALIAS -from .conv import conv1d_transpose #DEFINE_ALIAS -from .common import linear #DEFINE_ALIAS -from .conv import conv2d #DEFINE_ALIAS -from .conv import conv2d_transpose #DEFINE_ALIAS -from .conv import conv3d #DEFINE_ALIAS -from .conv import conv3d_transpose #DEFINE_ALIAS -# from .extension import add_position_encoding #DEFINE_ALIAS -# from .extension import autoincreased_step_counter #DEFINE_ALIAS -# from .extension import continuous_value_model #DEFINE_ALIAS -# from .extension import filter_by_instag #DEFINE_ALIAS -# from .extension import linear_chain_crf #DEFINE_ALIAS -# from .extension import merge_selected_rows #DEFINE_ALIAS -# from .extension import multiclass_nms #DEFINE_ALIAS -# from .extension import polygon_box_transform #DEFINE_ALIAS -# from .extension import random_crop #DEFINE_ALIAS -# from .extension import rpn_target_assign #DEFINE_ALIAS -# from .extension import similarity_focus #DEFINE_ALIAS -# from .extension import target_assign #DEFINE_ALIAS -# from .extension import temporal_shift #DEFINE_ALIAS -# from .extension import warpctc #DEFINE_ALIAS -from .extension import diag_embed #DEFINE_ALIAS +from .activation import elu # noqa: F401 +from .activation import elu_ # noqa: F401 +from .activation import gelu # noqa: F401 +from .activation import hardshrink # noqa: F401 +from .activation import hardtanh # noqa: F401 +from .activation import hardsigmoid # noqa: F401 +from .activation import hardswish # noqa: F401 +from .activation import leaky_relu # noqa: F401 +from .activation import log_sigmoid # noqa: F401 +from .activation import maxout # noqa: F401 +from .activation import prelu # noqa: F401 +from .activation import relu # noqa: F401 +from .activation import relu_ # noqa: F401 +from .activation import relu6 # noqa: F401 +from .activation import selu # noqa: F401 +from .activation import sigmoid # noqa: F401 +from .activation import silu # noqa: F401 +from .activation import softmax # noqa: F401 +from .activation import softmax_ # noqa: F401 +from .activation import softplus # noqa: F401 +from .activation import softshrink # noqa: F401 +from .activation import softsign # noqa: F401 +from .activation import swish # noqa: F401 +from .activation import tanh # noqa: F401 +from .activation import tanh_ # noqa: F401 +from .activation import tanhshrink # noqa: F401 +from .activation import thresholded_relu # noqa: F401 +from .activation import log_softmax # noqa: F401 +from .activation import glu # noqa: F401 +from .common import dropout # noqa: F401 +from .common import dropout2d # noqa: F401 +from .common import dropout3d # noqa: F401 +from .common import alpha_dropout # noqa: F401 +from .common import label_smooth # noqa: F401 +from .common import pad # noqa: F401 +from .common import cosine_similarity # noqa: F401 +from .common import unfold # noqa: F401 +from .common import interpolate # noqa: F401 +from .common import upsample # noqa: F401 +from .common import bilinear # noqa: F401 +from .conv import conv1d # noqa: F401 +from .conv import conv1d_transpose # noqa: F401 +from .common import linear # noqa: F401 +from .conv import conv2d # noqa: F401 +from .conv import conv2d_transpose # noqa: F401 +from .conv import conv3d # noqa: F401 +from .conv import conv3d_transpose # noqa: F401 +from .extension import diag_embed # noqa: F401 from .extension import sequence_mask -# from .lod import sequence_concat #DEFINE_ALIAS -# from .lod import sequence_conv #DEFINE_ALIAS -# from .lod import sequence_enumerate #DEFINE_ALIAS -# from .lod import sequence_expand_as #DEFINE_ALIAS -# from .lod import sequence_expand #DEFINE_ALIAS -# from .lod import sequence_first_step #DEFINE_ALIAS -# from .lod import sequence_last_step #DEFINE_ALIAS -# from .lod import sequence_mask #DEFINE_ALIAS -# from .lod import sequence_pad #DEFINE_ALIAS -# from .lod import sequence_pool #DEFINE_ALIAS -# from .lod import sequence_reshape #DEFINE_ALIAS -# from .lod import sequence_reverse #DEFINE_ALIAS -# from .lod import sequence_scatter #DEFINE_ALIAS -# from .lod import sequence_slice #DEFINE_ALIAS -# from .lod import sequence_softmax #DEFINE_ALIAS -# from .lod import sequence_unpad #DEFINE_ALIAS -# from .lod import array_length #DEFINE_ALIAS -# from .lod import array_read #DEFINE_ALIAS -# from .lod import array_write #DEFINE_ALIAS -# from .lod import create_array #DEFINE_ALIAS -# from .lod import hash #DEFINE_ALIAS -# from .lod import im2sequence #DEFINE_ALIAS -# from .lod import lod_append #DEFINE_ALIAS -# from .lod import lod_reset #DEFINE_ALIAS -# from .lod import reorder_lod_tensor_by_rank #DEFINE_ALIAS -# from .lod import tensor_array_to_tensor #DEFINE_ALIAS -# from .lod import dynamic_gru #DEFINE_ALIAS -# from .lod import dynamic_lstm #DEFINE_ALIAS -# from .lod import dynamic_lstmp #DEFINE_ALIAS -from .loss import binary_cross_entropy #DEFINE_ALIAS -from .loss import binary_cross_entropy_with_logits #DEFINE_ALIAS -# from .loss import bpr_loss #DEFINE_ALIAS -# from .loss import center_loss #DEFINE_ALIAS -#from .loss import cross_entropy #DEFINE_ALIAS -from .loss import cross_entropy #DEFINE_ALIAS -from .loss import dice_loss #DEFINE_ALIAS -from .loss import hsigmoid_loss #DEFINE_ALIAS -from .loss import kl_div #DEFINE_ALIAS -from .loss import l1_loss #DEFINE_ALIAS -from .loss import log_loss #DEFINE_ALIAS -from .loss import margin_ranking_loss #DEFINE_ALIAS -from .loss import mse_loss #DEFINE_ALIAS -from .loss import nll_loss #DEFINE_ALIAS -# from .loss import nce #DEFINE_ALIAS -from .loss import npair_loss #DEFINE_ALIAS -from .loss import sigmoid_focal_loss #DEFINE_ALIAS -# from .loss import smooth_l1 #DEFINE_ALIAS -from .loss import smooth_l1_loss #DEFINE_ALIAS -from .loss import softmax_with_cross_entropy #DEFINE_ALIAS -from .loss import square_error_cost #DEFINE_ALIAS -# from .loss import teacher_student_sigmoid_loss #DEFINE_ALIAS -from .loss import ctc_loss #DEFINE_ALIAS -# from .norm import data_norm #DEFINE_ALIAS -# from .norm import group_norm #DEFINE_ALIAS -from .norm import batch_norm #DEFINE_ALIAS -from .norm import instance_norm #DEFINE_ALIAS -from .norm import layer_norm #DEFINE_ALIAS -from .norm import local_response_norm #DEFINE_ALIAS -from .norm import normalize #DEFINE_ALIAS -# from .norm import spectral_norm #DEFINE_ALIAS -# from .pooling import pool2d #DEFINE_ALIAS -# from .pooling import pool3d #DEFINE_ALIAS -from .pooling import avg_pool1d #DEFINE_ALIAS -from .pooling import avg_pool2d #DEFINE_ALIAS -from .pooling import avg_pool3d #DEFINE_ALIAS -from .pooling import max_pool1d #DEFINE_ALIAS -from .pooling import max_pool2d #DEFINE_ALIAS -from .pooling import max_pool3d #DEFINE_ALIAS +from .loss import binary_cross_entropy # noqa: F401 +from .loss import binary_cross_entropy_with_logits # noqa: F401 +from .loss import cross_entropy # noqa: F401 +from .loss import dice_loss # noqa: F401 +from .loss import hsigmoid_loss # noqa: F401 +from .loss import kl_div # noqa: F401 +from .loss import l1_loss # noqa: F401 +from .loss import log_loss # noqa: F401 +from .loss import margin_ranking_loss # noqa: F401 +from .loss import mse_loss # noqa: F401 +from .loss import nll_loss # noqa: F401 +from .loss import npair_loss # noqa: F401 +from .loss import sigmoid_focal_loss # noqa: F401 +from .loss import smooth_l1_loss # noqa: F401 +from .loss import softmax_with_cross_entropy # noqa: F401 +from .loss import square_error_cost # noqa: F401 +from .loss import ctc_loss # noqa: F401 +from .norm import batch_norm # noqa: F401 +from .norm import instance_norm # noqa: F401 +from .norm import layer_norm # noqa: F401 +from .norm import local_response_norm # noqa: F401 +from .norm import normalize # noqa: F401 +from .pooling import avg_pool1d # noqa: F401 +from .pooling import avg_pool2d # noqa: F401 +from .pooling import avg_pool3d # noqa: F401 +from .pooling import max_pool1d # noqa: F401 +from .pooling import max_pool2d # noqa: F401 +from .pooling import max_pool3d # noqa: F401 -from .pooling import adaptive_max_pool1d #DEFINE_ALIAS -from .pooling import adaptive_max_pool2d #DEFINE_ALIAS -from .pooling import adaptive_max_pool3d #DEFINE_ALIAS -from .pooling import adaptive_avg_pool1d #DEFINE_ALIAS -from .pooling import adaptive_avg_pool2d #DEFINE_ALIAS -from .pooling import adaptive_avg_pool3d #DEFINE_ALIAS +from .pooling import adaptive_max_pool1d # noqa: F401 +from .pooling import adaptive_max_pool2d # noqa: F401 +from .pooling import adaptive_max_pool3d # noqa: F401 +from .pooling import adaptive_avg_pool1d # noqa: F401 +from .pooling import adaptive_avg_pool2d # noqa: F401 +from .pooling import adaptive_avg_pool3d # noqa: F401 -# from .rnn import rnn #DEFINE_ALIAS -# from .rnn import birnn #DEFINE_ALIAS -# from .rnn import gru_unit #DEFINE_ALIAS -# from .rnn import lstm #DEFINE_ALIAS -# from .rnn import lstm_unit #DEFINE_ALIAS -# from .vision import affine_channel #DEFINE_ALIAS -from .vision import affine_grid #DEFINE_ALIAS -# from .vision import anchor_generator #DEFINE_ALIAS -# from .vision import bipartite_match #DEFINE_ALIAS -# from .vision import box_clip #DEFINE_ALIAS -# from .vision import box_coder #DEFINE_ALIAS -# from .vision import box_decoder_and_assign #DEFINE_ALIAS -# from .vision import collect_fpn_proposals #DEFINE_ALIAS -# from .vision import deformable_conv #DEFINE_ALIAS -# from .vision import deformable_roi_pooling #DEFINE_ALIAS -# from .vision import density_prior_box #DEFINE_ALIAS -# from .vision import detection_output #DEFINE_ALIAS -# from .vision import distribute_fpn_proposals #DEFINE_ALIAS -# from .vision import fsp_matrix #DEFINE_ALIAS -# from .vision import generate_mask_labels #DEFINE_ALIAS -# from .vision import generate_proposal_labels #DEFINE_ALIAS -# from .vision import generate_proposals #DEFINE_ALIAS -from .vision import grid_sample #DEFINE_ALIAS -# from .vision import image_resize #DEFINE_ALIAS -# from .vision import image_resize_short #DEFINE_ALIAS -# from .vision import multi_box_head #DEFINE_ALIAS -from .vision import pixel_shuffle #DEFINE_ALIAS -# from .vision import prior_box #DEFINE_ALIAS -# from .vision import prroi_pool #DEFINE_ALIAS -# from .vision import psroi_pool #DEFINE_ALIAS -# from .vision import resize_bilinear #DEFINE_ALIAS -# from .vision import resize_nearest #DEFINE_ALIAS -# from .vision import resize_trilinear #DEFINE_ALIAS -# from .vision import retinanet_detection_output #DEFINE_ALIAS -# from .vision import retinanet_target_assign #DEFINE_ALIAS -# from .vision import roi_align #DEFINE_ALIAS -# from .vision import roi_perspective_transform #DEFINE_ALIAS -# from .vision import roi_pool #DEFINE_ALIAS -# from .vision import shuffle_channel #DEFINE_ALIAS -# from .vision import space_to_depth #DEFINE_ALIAS -# from .vision import yolo_box #DEFINE_ALIAS -# from .vision import yolov3_loss #DEFINE_ALIAS -from .input import one_hot #DEFINE_ALIAS -from .input import embedding #DEFINE_ALIAS -from ...fluid.layers import gather_tree -from ...fluid.layers import temporal_shift +from .vision import affine_grid # noqa: F401 +from .vision import grid_sample # noqa: F401 +from .vision import pixel_shuffle # noqa: F401 +from .input import one_hot # noqa: F401 +from .input import embedding # noqa: F401 +from ...fluid.layers import gather_tree # noqa: F401 +from ...fluid.layers import temporal_shift # noqa: F401 + +__all__ = [ #noqa + 'conv1d', + 'conv1d_transpose', + 'conv2d', + 'conv2d_transpose', + 'conv3d', + 'conv3d_transpose', + 'elu', + 'elu_', + 'gelu', + 'hardshrink', + 'hardtanh', + 'hardsigmoid', + 'hardswish', + 'leaky_relu', + 'log_sigmoid', + 'maxout', + 'prelu', + 'relu', + 'relu_', + 'relu6', + 'selu', + 'softmax', + 'softmax_', + 'softplus', + 'softshrink', + 'softsign', + 'sigmoid', + 'silu', + 'swish', + 'tanh', + 'tanh_', + 'tanhshrink', + 'thresholded_relu', + 'log_softmax', + 'glu', + 'diag_embed', + 'sequence_mask', + 'dropout', + 'dropout2d', + 'dropout3d', + 'alpha_dropout', + 'label_smooth', + 'linear', + 'pad', + 'unfold', + 'interpolate', + 'upsample', + 'bilinear', + 'cosine_similarity', + 'avg_pool1d', + 'avg_pool2d', + 'avg_pool3d', + 'max_pool1d', + 'max_pool2d', + 'max_pool3d', + 'adaptive_avg_pool1d', + 'adaptive_avg_pool2d', + 'adaptive_avg_pool3d', + 'adaptive_max_pool1d', + 'adaptive_max_pool2d', + 'adaptive_max_pool3d', + 'binary_cross_entropy', + 'binary_cross_entropy_with_logits', + 'cross_entropy', + 'dice_loss', + 'hsigmoid_loss', + 'kl_div', + 'l1_loss', + 'log_loss', + 'mse_loss', + 'margin_ranking_loss', + 'nll_loss', + 'npair_loss', + 'sigmoid_focal_loss', + 'smooth_l1_loss', + 'softmax_with_cross_entropy', + 'square_error_cost', + 'ctc_loss', + 'affine_grid', + 'grid_sample', + 'local_response_norm', + 'pixel_shuffle', + 'embedding', + 'gather_tree', + 'one_hot', + 'normalize' +] diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index d74308dc9aa32..cd8ee99baa237 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -12,53 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: define activation functions of neural network -from ...fluid.layers import brelu #DEFINE_ALIAS -# from ...fluid.layers import erf #DEFINE_ALIAS -from ...fluid.layers import maxout #DEFINE_ALIAS -# from ...fluid.layers import soft_relu #DEFINE_ALIAS -from ...fluid.layers import swish #DEFINE_ALIAS -from ...fluid.layers import sigmoid #DEFINE_ALIAS -from ...tensor.math import tanh #DEFINE_ALIAS -from ...tensor.math import tanh_ #DEFINE_ALIAS +from ...fluid.layers import sigmoid # noqa: F401 +from ...tensor.math import tanh # noqa: F401 +from ...tensor.math import tanh_ # noqa: F401 from ...tensor.manipulation import _print_warning_in_static_mode from ...tensor.manipulation import chunk from ...tensor.math import multiply -__all__ = [ - 'brelu', - 'elu', - 'elu_', - 'gelu', - 'hardshrink', - 'hardtanh', - 'hardsigmoid', - 'hardswish', - 'leaky_relu', - 'log_sigmoid', - 'maxout', - 'prelu', - 'relu', - 'relu_', - 'relu6', - 'selu', - 'softmax', - 'softmax_', - 'softplus', - 'softshrink', - 'softsign', - 'sigmoid', - 'silu' - 'swish', - 'tanh', - 'tanh_', - 'tanhshrink', - 'thresholded_relu', - 'log_softmax', - 'glu', -] - import warnings from ...fluid.layer_helper import LayerHelper from ...fluid.framework import in_dygraph_mode, convert_np_dtype_to_dtype_ diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index 1cc8ef6c39b15..7379c7a5f67bd 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -20,44 +20,20 @@ from ...fluid.layers import core from ...fluid import dygraph_utils # TODO: define the common functions to build a neural network -# from ...fluid import one_hot #DEFINE_ALIAS -# from ...fluid.layers import pad2d #DEFINE_ALIAS -from ...fluid.layers import unfold #DEFINE_ALIAS -from ...fluid.layers import squeeze #DEFINE_ALIAS -from ...fluid.layers import unsqueeze #DEFINE_ALIAS +from ...fluid.layers import unfold # noqa: F401 +from ...fluid.layers import squeeze +from ...fluid.layers import unsqueeze from ...tensor import clip from ...tensor import sum from ...tensor import sqrt -from ...tensor import sum #DEFINE_ALIAS -from ...tensor import sqrt #DEFINE_ALIAS from ...fluid.data_feeder import check_variable_and_dtype, check_dtype from ...fluid.framework import Variable, in_dygraph_mode, _varbase_creator -#from ...fluid.layers import fc #DEFINE_ALIAS -# from ...fluid.layers import pad_constant_like #DEFINE_ALIAS from ...fluid.framework import in_dygraph_mode from ...fluid import core, dygraph_utils from ...fluid import core, layers from ...fluid.data_feeder import check_variable_and_dtype -__all__ = [ - 'dropout', - 'dropout2d', - 'dropout3d', - 'alpha_dropout', - # 'embedding', - # 'fc', - 'label_smooth', - 'linear', - 'pad', - 'unfold', - # 'bilinear_tensor_product', - 'interpolate', - 'upsample', - 'bilinear', - 'cosine_similarity', -] - def interpolate(x, size=None, diff --git a/python/paddle/nn/functional/conv.py b/python/paddle/nn/functional/conv.py index a8d6a6cc38df2..800c820497372 100644 --- a/python/paddle/nn/functional/conv.py +++ b/python/paddle/nn/functional/conv.py @@ -13,15 +13,6 @@ # limitations under the License. from __future__ import print_function -__all__ = [ - 'conv1d', - 'conv1d_transpose', - 'conv2d', - 'conv2d_transpose', - 'conv3d', - 'conv3d_transpose', -] - import numpy as np from ...device import get_cudnn_version from ...fluid.framework import Variable, in_dygraph_mode diff --git a/python/paddle/nn/functional/extension.py b/python/paddle/nn/functional/extension.py index b004d79a877e7..7900f903e7fd2 100644 --- a/python/paddle/nn/functional/extension.py +++ b/python/paddle/nn/functional/extension.py @@ -14,8 +14,6 @@ # TODO: define the extention functions -__all__ = ['diag_embed', 'sequence_mask'] - import numpy as np from ...fluid.data_feeder import check_dtype from ...fluid.layer_helper import LayerHelper diff --git a/python/paddle/nn/functional/input.py b/python/paddle/nn/functional/input.py index b88a2b042ff48..4fff9cda4be33 100644 --- a/python/paddle/nn/functional/input.py +++ b/python/paddle/nn/functional/input.py @@ -19,8 +19,6 @@ from ...fluid.layers import core from ...fluid.data_feeder import check_variable_and_dtype, check_dtype -__all__ = ['one_hot', 'embedding'] - def one_hot(x, num_classes, name=None): """ diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index ca0ad06532d27..bb2d8005f4e31 100755 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -24,14 +24,14 @@ import paddle.fluid as fluid from ...fluid.framework import core, in_dygraph_mode from ...fluid.layers.nn import _elementwise_op_in_dygraph -from ...fluid.layers import dice_loss #DEFINE_ALIAS -from ...fluid.layers import log_loss #DEFINE_ALIAS -from ...fluid.layers import npair_loss #DEFINE_ALIAS +from ...fluid.layers import dice_loss # noqa: F401 +from ...fluid.layers import log_loss # noqa: F401 +from ...fluid.layers import npair_loss # noqa: F401 from ...fluid.layers import reshape -from ...fluid.layers import softmax_with_cross_entropy as fluid_softmax_with_cross_entropy #DEFINE_ALIAS -from ...fluid.layers import square_error_cost #DEFINE_ALIAS +from ...fluid.layers import softmax_with_cross_entropy as fluid_softmax_with_cross_entropy +from ...fluid.layers import square_error_cost # noqa: F401 -from ...fluid.layers import edit_distance #DEFINE_ALIAS +from ...fluid.layers import edit_distance # noqa: F401 from ...fluid.layers import huber_loss from ...fluid.layer_helper import LayerHelper from ...fluid.framework import in_dygraph_mode @@ -39,27 +39,6 @@ from ...fluid.framework import Variable from paddle.utils import deprecated -__all__ = [ - 'binary_cross_entropy', - 'binary_cross_entropy_with_logits', - 'cross_entropy', - 'dice_loss', - 'hsigmoid_loss', - 'kl_div', - 'l1_loss', - 'log_loss', - 'mse_loss', - 'margin_ranking_loss', - # 'nce', - 'nll_loss', - 'npair_loss', - 'sigmoid_focal_loss', - 'smooth_l1_loss', - 'softmax_with_cross_entropy', - 'square_error_cost', - 'ctc_loss', -] - def binary_cross_entropy(input, label, weight=None, reduction='mean', name=None): @@ -1312,7 +1291,7 @@ def cross_entropy(input, Indicate whether compute softmax before cross_entropy. Default is ``True``. - - **name** (str，optional) + - **name** (str, optional) The name of the operator. Default is ``None`` . For more information, please refer to :ref:`api_guide_Name` . diff --git a/python/paddle/nn/functional/norm.py b/python/paddle/nn/functional/norm.py index 73df03e3714c7..dddc4c66d591c 100644 --- a/python/paddle/nn/functional/norm.py +++ b/python/paddle/nn/functional/norm.py @@ -22,19 +22,8 @@ from ...fluid.initializer import Constant from ...fluid.param_attr import ParamAttr from ...fluid import core, dygraph_utils - import numbers -__all__ = [ - 'batch_norm', - # 'data_norm', - 'instance_norm', - 'layer_norm', - 'local_response_norm', - 'normalize', - # 'spectral_norm' -] - def normalize(x, p=2, axis=1, epsilon=1e-12, name=None): r""" diff --git a/python/paddle/nn/functional/pooling.py b/python/paddle/nn/functional/pooling.py index 5f3642710ae0a..27a66c629cafa 100755 --- a/python/paddle/nn/functional/pooling.py +++ b/python/paddle/nn/functional/pooling.py @@ -18,21 +18,6 @@ from ...fluid.layers import utils, LayerHelper, unsqueeze, squeeze from ...fluid.data_feeder import check_type, check_variable_and_dtype -__all__ = [ - 'avg_pool1d', - 'avg_pool2d', - 'avg_pool3d', - 'max_pool1d', - 'max_pool2d', - 'max_pool3d', - 'adaptive_avg_pool1d', - 'adaptive_avg_pool2d', - 'adaptive_avg_pool3d', - 'adaptive_max_pool1d', - 'adaptive_max_pool2d', - 'adaptive_max_pool3d', -] - def _is_list_or_tuple(input): return isinstance(input, (list, tuple)) diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index 032d5b47eda07..cb8a817023d22 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -19,43 +19,6 @@ from ...fluid import dygraph_utils import numpy as np -# TODO: define specitial functions used in computer vision task -# from ...fluid.layers import affine_channel #DEFINE_ALIAS -# from ...fluid.layers import anchor_generator #DEFINE_ALIAS -# from ...fluid.layers import bipartite_match #DEFINE_ALIAS -# from ...fluid.layers import box_clip #DEFINE_ALIAS -# from ...fluid.layers import box_coder #DEFINE_ALIAS -# from ...fluid.layers import box_decoder_and_assign #DEFINE_ALIAS -# from ...fluid.layers import collect_fpn_proposals #DEFINE_ALIAS -# from ...fluid.layers import deformable_roi_pooling #DEFINE_ALIAS -# from ...fluid.layers import density_prior_box #DEFINE_ALIAS -# from ...fluid.layers import detection_output #DEFINE_ALIAS -# from ...fluid.layers import distribute_fpn_proposals #DEFINE_ALIAS -# from ...fluid.layers import generate_mask_labels #DEFINE_ALIAS -# from ...fluid.layers import generate_proposal_labels #DEFINE_ALIAS -# from ...fluid.layers import generate_proposals #DEFINE_ALIAS -# from ...fluid.layers import image_resize #DEFINE_ALIAS -# from ...fluid.layers import prior_box #DEFINE_ALIAS -# from ...fluid.layers import prroi_pool #DEFINE_ALIAS -# from ...fluid.layers import psroi_pool #DEFINE_ALIAS -# from ...fluid.layers import resize_bilinear #DEFINE_ALIAS -# from ...fluid.layers import resize_nearest #DEFINE_ALIAS -# from ...fluid.layers import resize_trilinear #DEFINE_ALIAS -# from ...fluid.layers import roi_align #DEFINE_ALIAS -# from ...fluid.layers import roi_pool #DEFINE_ALIAS -# from ...fluid.layers import space_to_depth #DEFINE_ALIAS -# from ...fluid.layers import yolo_box #DEFINE_ALIAS -# from ...fluid.layers import yolov3_loss #DEFINE_ALIAS -# from ...fluid.layers import fsp_matrix #DEFINE_ALIAS -# from ...fluid.layers import image_resize_short #DEFINE_ALIAS -# from ...fluid.layers import pixel_shuffle #DEFINE_ALIAS -# from ...fluid.layers import retinanet_detection_output #DEFINE_ALIAS -# from ...fluid.layers import retinanet_target_assign #DEFINE_ALIAS -# from ...fluid.layers import roi_perspective_transform #DEFINE_ALIAS -# from ...fluid.layers import shuffle_channel #DEFINE_ALIAS - -__all__ = ['affine_grid', 'grid_sample', 'pixel_shuffle'] - def affine_grid(theta, out_shape, align_corners=True, name=None): """ diff --git a/python/paddle/nn/initializer/__init__.py b/python/paddle/nn/initializer/__init__.py index c128a1b401b2d..03e91f80dd139 100644 --- a/python/paddle/nn/initializer/__init__.py +++ b/python/paddle/nn/initializer/__init__.py @@ -13,36 +13,34 @@ # limitations under the License. # TODO: define the initializers to create a Parameter in neural network -from ...fluid.initializer import Bilinear #DEFINE_ALIAS -from ...fluid.initializer import set_global_initializer #DEFINE_ALIAS +from ...fluid.initializer import Bilinear # noqa: F401 +from ...fluid.initializer import set_global_initializer # noqa: F401 -from . import constant -from .constant import Constant #DEFINE_ALIAS +from .constant import Constant # noqa: F401 -from . import kaiming -from .kaiming import KaimingNormal #DEFINE_ALIAS -from .kaiming import KaimingUniform #DEFINE_ALIAS +from .kaiming import KaimingNormal # noqa: F401 +from .kaiming import KaimingUniform # noqa: F401 -__all__ = ['Bilinear', 'set_global_initializer'] +from .xavier import XavierNormal # noqa: F401 +from .xavier import XavierUniform # noqa: F401 -__all__ += constant.__all__ -__all__ += kaiming.__all__ +from .assign import Assign # noqa: F401 -from . import xavier -from .xavier import XavierNormal #DEFINE_ALIAS -from .xavier import XavierUniform #DEFINE_ALIAS +from .normal import Normal # noqa: F401 +from .normal import TruncatedNormal # noqa: F401 -from . import assign -from .assign import Assign #DEFINE_ALIAS +from .uniform import Uniform # noqa: F401 -from . import normal -from .normal import Normal #DEFINE_ALIAS -from .normal import TruncatedNormal #DEFINE_ALIAS - -from . import uniform -from .uniform import Uniform #DEFINE_ALIAS - -__all__ += xavier.__all__ -__all__ += assign.__all__ -__all__ += normal.__all__ -__all__ += uniform.__all__ +__all__ = [ #noqa + 'Bilinear', + 'Constant', + 'KaimingUniform', + 'KaimingNormal', + 'XavierNormal', + 'XavierUniform', + 'Assign', + 'Normal', + 'TruncatedNormal', + 'Uniform', + 'set_global_initializer' +] diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py index 94c4ddc193882..642919f354075 100644 --- a/python/paddle/nn/initializer/assign.py +++ b/python/paddle/nn/initializer/assign.py @@ -19,8 +19,6 @@ from ...fluid.data_feeder import check_type from ...fluid.initializer import NumpyArrayInitializer -__all__ = ['Assign'] - class Assign(NumpyArrayInitializer): """Init an parameter with a numpy array, list, or tensor. diff --git a/python/paddle/nn/initializer/constant.py b/python/paddle/nn/initializer/constant.py index 6d21ddae0d16b..aec3e82aab62b 100644 --- a/python/paddle/nn/initializer/constant.py +++ b/python/paddle/nn/initializer/constant.py @@ -15,8 +15,6 @@ # TODO: define the initializers of Constant in neural network from ...fluid.initializer import ConstantInitializer -__all__ = ['Constant'] - class Constant(ConstantInitializer): """Implement the constant initializer. diff --git a/python/paddle/nn/initializer/kaiming.py b/python/paddle/nn/initializer/kaiming.py index 7e2b6f787f853..712bffccda102 100644 --- a/python/paddle/nn/initializer/kaiming.py +++ b/python/paddle/nn/initializer/kaiming.py @@ -15,8 +15,6 @@ # TODO: define the initializers of Kaiming functions in neural network from ...fluid.initializer import MSRAInitializer -__all__ = ['KaimingUniform', 'KaimingNormal'] - class KaimingNormal(MSRAInitializer): r"""Implements the Kaiming Normal initializer diff --git a/python/paddle/nn/initializer/normal.py b/python/paddle/nn/initializer/normal.py index a572d0e2c9216..c009df780054e 100644 --- a/python/paddle/nn/initializer/normal.py +++ b/python/paddle/nn/initializer/normal.py @@ -15,8 +15,6 @@ from ...fluid.initializer import NormalInitializer from ...fluid.initializer import TruncatedNormalInitializer -__all__ = ['Normal', 'TruncatedNormal'] - class Normal(NormalInitializer): """The Random Normal (Gaussian) distribution initializer. diff --git a/python/paddle/nn/initializer/uniform.py b/python/paddle/nn/initializer/uniform.py index a5d7d34efcf66..e54a4d2187b8d 100644 --- a/python/paddle/nn/initializer/uniform.py +++ b/python/paddle/nn/initializer/uniform.py @@ -14,8 +14,6 @@ from ...fluid.initializer import UniformInitializer -__all__ = ['Uniform'] - class Uniform(UniformInitializer): """The random uniform distribution initializer. diff --git a/python/paddle/nn/initializer/xavier.py b/python/paddle/nn/initializer/xavier.py index 821a698475310..01a4a8887b489 100644 --- a/python/paddle/nn/initializer/xavier.py +++ b/python/paddle/nn/initializer/xavier.py @@ -14,8 +14,6 @@ from ...fluid.initializer import XavierInitializer -__all__ = ['XavierNormal', 'XavierUniform'] - class XavierNormal(XavierInitializer): r""" diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py index 17c4ca5c5d11d..64f0391fb6533 100644 --- a/python/paddle/nn/layer/__init__.py +++ b/python/paddle/nn/layer/__init__.py @@ -14,90 +14,70 @@ # TODO: define activation functions of neural network -from . import activation -from . import loss -from . import conv -from . import activation -from . import norm -from . import rnn -from . import vision -from . import distance -from . import transformer -from . import container +from . import rnn # noqa: F401 +from . import transformer # noqa: F401 +from . import container # noqa: F401 -from .activation import * -from .loss import * -from .conv import * -from .activation import * -from .norm import * -from .rnn import * -from .vision import * +from .activation import PReLU # noqa: F401 +from .activation import ReLU # noqa: F401 +from .activation import ReLU6 # noqa: F401 +from .activation import LeakyReLU # noqa: F401 +from .activation import Sigmoid # noqa: F401 +from .activation import Softmax # noqa: F401 +from .activation import LogSoftmax # noqa: F401 +from .common import Bilinear # noqa: F401 +from .common import Pad1D # noqa: F401 +from .common import Pad2D # noqa: F401 +from .common import Pad3D # noqa: F401 +from .common import CosineSimilarity # noqa: F401 +from .common import Embedding # noqa: F401 +from .common import Linear # noqa: F401 +from .common import Flatten # noqa: F401 +from .common import Upsample # noqa: F401 +from .common import Dropout # noqa: F401 +from .common import Dropout2D # noqa: F401 +from .common import Dropout3D # noqa: F401 +from .common import AlphaDropout # noqa: F401 +from .common import Upsample # noqa: F401 +from .common import UpsamplingBilinear2D # noqa: F401 +from .common import UpsamplingNearest2D # noqa: F401 +from .pooling import AvgPool1D # noqa: F401 +from .pooling import AvgPool2D # noqa: F401 +from .pooling import AvgPool3D # noqa: F401 +from .pooling import MaxPool1D # noqa: F401 +from .pooling import MaxPool2D # noqa: F401 +from .pooling import MaxPool3D # noqa: F401 +from .pooling import AdaptiveAvgPool1D # noqa: F401 +from .pooling import AdaptiveAvgPool2D # noqa: F401 +from .pooling import AdaptiveAvgPool3D # noqa: F401 +from .pooling import AdaptiveMaxPool1D # noqa: F401 +from .pooling import AdaptiveMaxPool2D # noqa: F401 +from .pooling import AdaptiveMaxPool3D # noqa: F401 +from .conv import Conv1D # noqa: F401 +from .conv import Conv2D # noqa: F401 +from .conv import Conv3D # noqa: F401 +from .conv import Conv1DTranspose # noqa: F401 +from .conv import Conv2DTranspose # noqa: F401 +from .conv import Conv3DTranspose # noqa: F401 +from .loss import BCEWithLogitsLoss # noqa: F401 +from .loss import CrossEntropyLoss # noqa: F401 +from .loss import MSELoss # noqa: F401 +from .loss import L1Loss # noqa: F401 +from .loss import NLLLoss # noqa: F401 +from .loss import BCELoss # noqa: F401 +from .loss import KLDivLoss # noqa: F401 +from .loss import MarginRankingLoss # noqa: F401 +from .loss import CTCLoss # noqa: F401 +from .loss import SmoothL1Loss # noqa: F401 +from .norm import BatchNorm1D # noqa: F401 +from .norm import BatchNorm2D # noqa: F401 +from .norm import BatchNorm3D # noqa: F401 +from .norm import SyncBatchNorm # noqa: F401 +from .norm import GroupNorm # noqa: F401 +from .norm import LayerNorm # noqa: F401 +from .norm import SpectralNorm # noqa: F401 +from .norm import LocalResponseNorm # noqa: F401 -from .transformer import * -from .activation import PReLU #DEFINE_ALIAS -from .activation import ReLU #DEFINE_ALIAS -from .activation import LeakyReLU #DEFINE_ALIAS -from .activation import Sigmoid #DEFINE_ALIAS -from .activation import Softmax #DEFINE_ALIAS -from .activation import LogSoftmax #DEFINE_ALIAS -from .common import Bilinear #DEFINE_ALIAS -from .common import Pad1D #DEFINE_ALIAS -from .common import Pad2D #DEFINE_ALIAS -from .common import Pad3D #DEFINE_ALIAS -from .common import CosineSimilarity #DEFINE_ALIAS -from .common import Embedding #DEFINE_ALIAS -from .common import Linear #DEFINE_ALIAS -from .common import Flatten #DEFINE_ALIAS -from .common import Upsample #DEFINE_ALIAS -from .common import Dropout #DEFINE_ALIAS -from .common import Dropout2D #DEFINE_ALIAS -from .common import Dropout3D #DEFINE_ALIAS -from .common import AlphaDropout #DEFINE_ALIAS -from .common import Upsample #DEFINE_ALIAS -from .common import UpsamplingBilinear2D #DEFINE_ALIAS -from .common import UpsamplingNearest2D #DEFINE_ALIAS -from .pooling import AvgPool1D #DEFINE_ALIAS -from .pooling import AvgPool2D #DEFINE_ALIAS -from .pooling import AvgPool3D #DEFINE_ALIAS -from .pooling import MaxPool1D #DEFINE_ALIAS -from .pooling import MaxPool2D #DEFINE_ALIAS -from .pooling import MaxPool3D #DEFINE_ALIAS -from .pooling import AdaptiveAvgPool1D #DEFINE_ALIAS -from .pooling import AdaptiveAvgPool2D #DEFINE_ALIAS -from .pooling import AdaptiveAvgPool3D #DEFINE_ALIAS -from .pooling import AdaptiveMaxPool1D #DEFINE_ALIAS -from .pooling import AdaptiveMaxPool2D #DEFINE_ALIAS -from .pooling import AdaptiveMaxPool3D #DEFINE_ALIAS -from .conv import Conv1D #DEFINE_ALIAS -from .conv import Conv2D #DEFINE_ALIAS -from .conv import Conv3D #DEFINE_ALIAS -from .conv import Conv1DTranspose #DEFINE_ALIAS -from .conv import Conv2DTranspose #DEFINE_ALIAS -from .conv import Conv3DTranspose #DEFINE_ALIAS -# from .conv import TreeConv #DEFINE_ALIAS -# from .conv import Conv1D #DEFINE_ALIAS -# from .loss import NCELoss #DEFINE_ALIAS -from .loss import BCEWithLogitsLoss #DEFINE_ALIAS -from .loss import CrossEntropyLoss #DEFINE_ALIAS -from .loss import MSELoss #DEFINE_ALIAS -from .loss import L1Loss #DEFINE_ALIAS -from .loss import NLLLoss #DEFINE_ALIAS -from .loss import BCELoss #DEFINE_ALIAS -from .loss import KLDivLoss #DEFINE_ALIAS -from .loss import MarginRankingLoss #DEFINE_ALIAS -from .loss import CTCLoss #DEFINE_ALIAS -from .loss import SmoothL1Loss #DEFINE_ALIAS -from .norm import BatchNorm #DEFINE_ALIAS -from .norm import SyncBatchNorm #DEFINE_ALIAS -from .norm import GroupNorm #DEFINE_ALIAS -from .norm import LayerNorm #DEFINE_ALIAS -from .norm import SpectralNorm #DEFINE_ALIAS -#from .norm import InstanceNorm #DEFINE_ALIAS -from .norm import LocalResponseNorm #DEFINE_ALIAS -# from .rnn import RNNCell #DEFINE_ALIAS -# from .rnn import GRUCell #DEFINE_ALIAS -# from .rnn import LSTMCell #DEFINE_ALIAS - -from .vision import PixelShuffle #DEFINE_ALIAS -from .distance import PairwiseDistance #DEFINE_ALIAS -from .container import LayerDict #DEFINE_ALIAS +from .vision import PixelShuffle # noqa: F401 +from .distance import PairwiseDistance # noqa: F401 +from .container import LayerDict # noqa: F401 diff --git a/python/paddle/nn/layer/activation.py b/python/paddle/nn/layer/activation.py index 2a9ae310615ce..c6ce4588ea5da 100644 --- a/python/paddle/nn/layer/activation.py +++ b/python/paddle/nn/layer/activation.py @@ -14,33 +14,6 @@ # TODO: define activation functions of neural network -__all__ = [ - 'ELU', - 'GELU', - 'Hardshrink', - 'Hardswish', - 'Tanh', - 'Hardtanh', - 'PReLU', - 'ReLU', - 'ReLU6', - 'SELU', - 'LeakyReLU', - 'Sigmoid', - 'Silu', - 'Hardsigmoid', - 'Softmax', - 'Softplus', - 'Softshrink', - 'Softsign', - 'Swish', - 'Tanhshrink', - 'ThresholdedReLU', - 'LogSigmoid', - 'LogSoftmax', - 'Maxout', -] - from ...fluid.dygraph import layers from ...fluid import core from ...fluid.framework import in_dygraph_mode diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py index 8c001793715e5..058507ba5dec3 100644 --- a/python/paddle/nn/layer/common.py +++ b/python/paddle/nn/layer/common.py @@ -14,30 +14,12 @@ # TODO: define the common classes to build a neural network import paddle -from ...fluid.dygraph import Flatten #DEFINE_ALIAS +from ...fluid.dygraph import Flatten # noqa: F401 from ...fluid.dygraph import layers from ...fluid.framework import in_dygraph_mode from .. import functional as F from ...fluid.framework import _dygraph_tracer -__all__ = [ - 'Embedding', - 'Linear', - 'Upsample', - 'Pad1D', - 'Pad2D', - 'Pad3D', - 'UpsamplingNearest2D', - 'UpsamplingBilinear2D', - 'CosineSimilarity', - 'Dropout', - 'Dropout2D', - 'Dropout3D', - 'Bilinear', - 'AlphaDropout', - 'Unfold', -] - def _npairs(x, n): if isinstance(x, (paddle.Tensor, list)): diff --git a/python/paddle/nn/layer/conv.py b/python/paddle/nn/layer/conv.py index d6ba04dad04c7..2360dc17cf171 100644 --- a/python/paddle/nn/layer/conv.py +++ b/python/paddle/nn/layer/conv.py @@ -14,15 +14,6 @@ # TODO: define classes of convolutional neural network -__all__ = [ - 'Conv1D', - 'Conv2D', - 'Conv3D', - 'Conv1DTranspose', - 'Conv2DTranspose', - 'Conv3DTranspose', -] - import numpy as np from ...fluid import get_flags diff --git a/python/paddle/nn/layer/distance.py b/python/paddle/nn/layer/distance.py index 72e0a1b2d6d20..7eb0fc1fbb575 100644 --- a/python/paddle/nn/layer/distance.py +++ b/python/paddle/nn/layer/distance.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['PairwiseDistance'] - import numpy as np import paddle diff --git a/python/paddle/nn/layer/loss.py b/python/paddle/nn/layer/loss.py index 2dfb3acca68e1..356b22c632cf5 100644 --- a/python/paddle/nn/layer/loss.py +++ b/python/paddle/nn/layer/loss.py @@ -21,20 +21,6 @@ from .. import functional as F from paddle.fluid.framework import core, in_dygraph_mode, _varbase_creator -__all__ = [ - 'BCEWithLogitsLoss', - 'CrossEntropyLoss', - 'HSigmoidLoss', - 'MSELoss', - 'L1Loss', - 'NLLLoss', - 'BCELoss', - 'KLDivLoss', - 'MarginRankingLoss', - 'CTCLoss', - 'SmoothL1Loss', -] - class BCEWithLogitsLoss(fluid.dygraph.Layer): r""" @@ -295,7 +281,7 @@ class CrossEntropyLoss(fluid.dygraph.Layer): Indicate whether compute softmax before cross_entropy. Default is ``True``. - - **name** (str，optional) + - **name** (str, optional) The name of the operator. Default is ``None`` . For more information, please refer to :ref:`api_guide_Name` . @@ -318,7 +304,7 @@ class CrossEntropyLoss(fluid.dygraph.Layer): - **label** (Tensor) - 1. If soft_label=False，the shape is + 1. If soft_label=False, the shape is :math:`[N_1, N_2, ..., N_k]` or :math:`[N_1, N_2, ..., N_k, 1]`, k >= 1. the data type is int32, int64, float32, float64, where each value is [0, C-1]. diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py index 0b0b2bf7b9b27..970d68e826343 100644 --- a/python/paddle/nn/layer/norm.py +++ b/python/paddle/nn/layer/norm.py @@ -28,13 +28,10 @@ # TODO: define normalization api import six -#from ...fluid.dygraph.nn import InstanceNorm -from ...fluid.dygraph import BatchNorm #DEFINE_ALIAS -#from ...fluid.dygraph import GroupNorm #DEFINE_ALIAS +from ...fluid.dygraph import BatchNorm # noqa: F401 -#from ...fluid.dygraph import LayerNorm #DEFINE_ALIAS -from ...fluid.dygraph import SpectralNorm #DEFINE_ALIAS +from ...fluid.dygraph import SpectralNorm # noqa: F401 from ...fluid.dygraph import layers from ...framework import get_default_dtype, set_default_dtype @@ -53,12 +50,6 @@ from ...fluid.dygraph.base import no_grad from .. import functional as F -__all__ = [ - 'BatchNorm', 'GroupNorm', 'LayerNorm', 'SpectralNorm', 'BatchNorm1D', - 'BatchNorm2D', 'BatchNorm3D', 'InstanceNorm1D', 'InstanceNorm2D', - 'InstanceNorm3D', 'SyncBatchNorm', 'LocalResponseNorm' -] - class _InstanceNormBase(layers.Layer): """ diff --git a/python/paddle/nn/layer/pooling.py b/python/paddle/nn/layer/pooling.py index cdb87a1cb3920..5916fd7c69eb0 100755 --- a/python/paddle/nn/layer/pooling.py +++ b/python/paddle/nn/layer/pooling.py @@ -16,21 +16,6 @@ from ...fluid.layer_helper import LayerHelper from .. import functional as F -__all__ = [ - 'AvgPool1D', - 'AvgPool2D', - 'AvgPool3D', - 'MaxPool1D', - 'MaxPool2D', - 'MaxPool3D', - 'AdaptiveAvgPool1D', - 'AdaptiveAvgPool2D', - 'AdaptiveAvgPool3D', - 'AdaptiveMaxPool1D', - 'AdaptiveMaxPool2D', - 'AdaptiveMaxPool3D', -] - class AvgPool1D(layers.Layer): r""" diff --git a/python/paddle/nn/layer/rnn.py b/python/paddle/nn/layer/rnn.py index 964cfa74ebf08..a7539b5b09571 100644 --- a/python/paddle/nn/layer/rnn.py +++ b/python/paddle/nn/layer/rnn.py @@ -33,18 +33,6 @@ from paddle.fluid.layers.utils import map_structure, flatten, pack_sequence_as from paddle.fluid.data_feeder import convert_dtype -__all__ = [ - 'RNNCellBase', - 'SimpleRNNCell', - 'LSTMCell', - 'GRUCell', - 'RNN', - 'BiRNN', - 'SimpleRNN', - 'LSTM', - 'GRU', -] - def split_states(states, bidirectional=False, state_components=1): r""" diff --git a/python/paddle/nn/layer/transformer.py b/python/paddle/nn/layer/transformer.py index fe70a99ffb518..752870f3d0a28 100644 --- a/python/paddle/nn/layer/transformer.py +++ b/python/paddle/nn/layer/transformer.py @@ -13,14 +13,6 @@ # limitations under the License. # TODO: define the classes of Transformer neural network -__all__ = [ - 'MultiHeadAttention', - 'TransformerEncoderLayer', - 'TransformerEncoder', - 'TransformerDecoderLayer', - 'TransformerDecoder', - 'Transformer', -] import copy import collections diff --git a/python/paddle/nn/layer/vision.py b/python/paddle/nn/layer/vision.py index d9c948a848a93..e66e122be5259 100644 --- a/python/paddle/nn/layer/vision.py +++ b/python/paddle/nn/layer/vision.py @@ -17,8 +17,6 @@ from ...fluid.dygraph import layers from .. import functional -__all__ = ['PixelShuffle'] - class PixelShuffle(layers.Layer): """ diff --git a/python/paddle/nn/utils/__init__.py b/python/paddle/nn/utils/__init__.py index 6562ac35e1e31..bf2573d2cbc2d 100644 --- a/python/paddle/nn/utils/__init__.py +++ b/python/paddle/nn/utils/__init__.py @@ -12,5 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import weight_norm_hook -from .weight_norm_hook import weight_norm, remove_weight_norm +from .weight_norm_hook import weight_norm, remove_weight_norm # noqa: F401 + +__all__ = [ #noqa + 'weight_norm', 'remove_weight_norm' +] diff --git a/python/paddle/nn/utils/weight_norm_hook.py b/python/paddle/nn/utils/weight_norm_hook.py index fdf7a1b5bb2e2..23df38ca08c45 100755 --- a/python/paddle/nn/utils/weight_norm_hook.py +++ b/python/paddle/nn/utils/weight_norm_hook.py @@ -19,8 +19,6 @@ from ...fluid.layer_helper import LayerHelper from ...fluid.data_feeder import check_variable_and_dtype -__all__ = ['weight_norm', 'remove_weight_norm'] - def l2_norm(x, axis, epsilon=1e-12, name=None): if len(x.shape) == 1: diff --git a/python/paddle/utils/deprecated.py b/python/paddle/utils/deprecated.py index daa2826ca360f..a46f1ae3a2c2e 100755 --- a/python/paddle/utils/deprecated.py +++ b/python/paddle/utils/deprecated.py @@ -83,13 +83,14 @@ def wrapper(*args, **kwargs): 2. since version is empty, in this case, API is deprecated in all versions. 3. current version is newer than since version. """ - msg = "\033[93mWarning %s \033[0m" % (msg) + warningmsg = "\033[93mWarning %s \033[0m" % (msg) v_current = [int(i) for i in paddle.__version__.split(".")] v_current += [0] * (4 - len(v_current)) v_since = [int(i) for i in _since.split(".")] v_since += [0] * (4 - len(v_since)) if paddle.__version__ == "0.0.0" or _since == "" or v_current >= v_since: - warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + warnings.warn( + warningmsg, category=DeprecationWarning, stacklevel=2) return func(*args, **kwargs) From 32203c38a63a420401151071fe72110e6f56caeb Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Wed, 28 Apr 2021 13:40:02 +0800 Subject: [PATCH 003/156] update 2.0 public api in paddle.init (#32034) (#32620) Co-authored-by: XiaoguangHu <46782768+XiaoguangHu01@users.noreply.github.com> Co-authored-by: XiaoguangHu <46782768+XiaoguangHu01@users.noreply.github.com> --- python/paddle/__init__.py | 712 ++++++++++++++++++++++++-------------- 1 file changed, 450 insertions(+), 262 deletions(-) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 4b9f310e73bbe..054fcdfcbe651 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import os - try: from paddle.version import full_version as __version__ from paddle.version import commit as __git_commit__ @@ -30,280 +27,471 @@ from .fluid.dygraph import monkey_patch_math_varbase monkey_patch_variable() monkey_patch_math_varbase() -import paddle.framework -from .framework.dtype import dtype as dtype -from paddle.framework.dtype import uint8 -from paddle.framework.dtype import int8 -from paddle.framework.dtype import int16 -from paddle.framework.dtype import int32 -from paddle.framework.dtype import int64 -from paddle.framework.dtype import float16 -from paddle.framework.dtype import float32 -from paddle.framework.dtype import float64 -from paddle.framework.dtype import bfloat16 -from paddle.framework.dtype import bool -from paddle.framework.dtype import complex64 -from paddle.framework.dtype import complex128 -from .framework import VarBase as Tensor -Tensor.__qualname__ = 'Tensor' -import paddle.compat -import paddle.distributed -import paddle.sysconfig -import paddle.tensor -import paddle.distribution -import paddle.nn -import paddle.distributed.fleet -import paddle.optimizer -import paddle.metric -import paddle.device -import paddle.regularizer -import paddle.incubate -import paddle.autograd +from .framework.dtype import dtype as dtype # noqa: F401 +from paddle.framework.dtype import uint8 # noqa: F401 +from paddle.framework.dtype import int8 # noqa: F401 +from paddle.framework.dtype import int16 # noqa: F401 +from paddle.framework.dtype import int32 # noqa: F401 +from paddle.framework.dtype import int64 # noqa: F401 +from paddle.framework.dtype import float16 # noqa: F401 +from paddle.framework.dtype import float32 # noqa: F401 +from paddle.framework.dtype import float64 # noqa: F401 +from paddle.framework.dtype import bfloat16 # noqa: F401 +from paddle.framework.dtype import bool # noqa: F401 +from paddle.framework.dtype import complex64 # noqa: F401 +from paddle.framework.dtype import complex128 # noqa: F401 +from .framework import VarBase as Tensor # noqa: F401 +Tensor.__qualname__ = 'Tensor' # noqa: F401 +import paddle.compat # noqa: F401 +import paddle.distributed # noqa: F401 +import paddle.sysconfig # noqa: F401 +import paddle.distribution # noqa: F401 +import paddle.nn # noqa: F401 +import paddle.distributed.fleet # noqa: F401 +import paddle.optimizer # noqa: F401 +import paddle.metric # noqa: F401 +import paddle.regularizer # noqa: F401 +import paddle.incubate # noqa: F401 +import paddle.autograd # noqa: F401 -# TODO: define alias in tensor and framework directory +import paddle.jit # noqa: F401 +import paddle.amp # noqa: F401 +import paddle.dataset # noqa: F401 +import paddle.inference # noqa: F401 +import paddle.io # noqa: F401 +import paddle.onnx # noqa: F401 +import paddle.reader # noqa: F401 +import paddle.static # noqa: F401 +import paddle.vision # noqa: F401 -from .tensor.random import randperm -from .tensor.random import bernoulli +from .tensor.random import bernoulli # noqa: F401 -from .tensor.attribute import rank #DEFINE_ALIAS -from .tensor.attribute import shape #DEFINE_ALIAS -from .tensor.attribute import real #DEFINE_ALIAS -from .tensor.attribute import imag #DEFINE_ALIAS -from .tensor.creation import to_tensor #DEFINE_ALIAS -from .tensor.creation import diag #DEFINE_ALIAS -from .tensor.creation import eye #DEFINE_ALIAS -# from .tensor.creation import fill_constant #DEFINE_ALIAS -# from .tensor.creation import get_tensor_from_selected_rows #DEFINE_ALIAS -from .tensor.creation import linspace #DEFINE_ALIAS -from .tensor.creation import ones #DEFINE_ALIAS -from .tensor.creation import ones_like #DEFINE_ALIAS -from .tensor.creation import zeros #DEFINE_ALIAS -from .tensor.creation import zeros_like #DEFINE_ALIAS -from .tensor.creation import arange #DEFINE_ALIAS -from .tensor.creation import eye #DEFINE_ALIAS -from .tensor.creation import full #DEFINE_ALIAS -from .tensor.creation import full_like #DEFINE_ALIAS -from .tensor.creation import triu #DEFINE_ALIAS -from .tensor.creation import tril #DEFINE_ALIAS -from .tensor.creation import meshgrid #DEFINE_ALIAS -from .tensor.creation import empty #DEFINE_ALIAS -from .tensor.creation import empty_like #DEFINE_ALIAS -from .tensor.creation import assign #DEFINE_ALIAS -from .tensor.linalg import matmul #DEFINE_ALIAS -from .tensor.linalg import dot #DEFINE_ALIAS -# from .tensor.linalg import einsum #DEFINE_ALIAS -from .tensor.linalg import norm #DEFINE_ALIAS -from .tensor.linalg import transpose #DEFINE_ALIAS -from .tensor.linalg import dist #DEFINE_ALIAS -from .tensor.linalg import t #DEFINE_ALIAS -from .tensor.linalg import cross #DEFINE_ALIAS -from .tensor.linalg import cholesky #DEFINE_ALIAS -# from .tensor.linalg import tensordot #DEFINE_ALIAS -from .tensor.linalg import bmm #DEFINE_ALIAS -from .tensor.linalg import histogram #DEFINE_ALIAS -from .tensor.linalg import mv #DEFINE_ALIAS -from .tensor.logic import equal #DEFINE_ALIAS -from .tensor.logic import greater_equal #DEFINE_ALIAS -from .tensor.logic import greater_than #DEFINE_ALIAS -from .tensor.logic import is_empty #DEFINE_ALIAS -#from .tensor.logic import isfinite #DEFINE_ALIAS -from .tensor.logic import less_equal #DEFINE_ALIAS -from .tensor.logic import less_than #DEFINE_ALIAS -from .tensor.logic import logical_and #DEFINE_ALIAS -from .tensor.logic import logical_not #DEFINE_ALIAS -from .tensor.logic import logical_or #DEFINE_ALIAS -from .tensor.logic import logical_xor #DEFINE_ALIAS -from .tensor.logic import not_equal #DEFINE_ALIAS -from .tensor.logic import allclose #DEFINE_ALIAS -from .tensor.logic import equal_all #DEFINE_ALIAS -# from .tensor.logic import isnan #DEFINE_ALIAS -from .tensor.logic import is_tensor #DEFINE_ALIAS -from .tensor.manipulation import cast #DEFINE_ALIAS -from .tensor.manipulation import concat #DEFINE_ALIAS -from .tensor.manipulation import expand #DEFINE_ALIAS -from .tensor.manipulation import broadcast_to #DEFINE_ALIAS -from .tensor.manipulation import expand_as #DEFINE_ALIAS -from .tensor.manipulation import tile #DEFINE_ALIAS -from .tensor.manipulation import flatten #DEFINE_ALIAS -from .tensor.manipulation import gather #DEFINE_ALIAS -from .tensor.manipulation import gather_nd #DEFINE_ALIAS -from .tensor.manipulation import reshape #DEFINE_ALIAS -from .tensor.manipulation import reshape_ #DEFINE_ALIAS -from .tensor.manipulation import flip as reverse #DEFINE_ALIAS -from .tensor.manipulation import scatter #DEFINE_ALIAS -from .tensor.manipulation import scatter_ #DEFINE_ALIAS -from .tensor.manipulation import scatter_nd_add #DEFINE_ALIAS -from .tensor.manipulation import scatter_nd #DEFINE_ALIAS -from .tensor.manipulation import shard_index #DEFINE_ALIAS -from .tensor.manipulation import slice #DEFINE_ALIAS -from .tensor.manipulation import split #DEFINE_ALIAS -from .tensor.manipulation import squeeze #DEFINE_ALIAS -from .tensor.manipulation import squeeze_ #DEFINE_ALIAS -from .tensor.manipulation import stack #DEFINE_ALIAS -from .tensor.manipulation import strided_slice #DEFINE_ALIAS -from .tensor.manipulation import transpose #DEFINE_ALIAS -from .tensor.manipulation import unique #DEFINE_ALIAS -from .tensor.manipulation import unsqueeze #DEFINE_ALIAS -from .tensor.manipulation import unsqueeze_ #DEFINE_ALIAS -from .tensor.manipulation import unstack #DEFINE_ALIAS -from .tensor.manipulation import flip #DEFINE_ALIAS -from .tensor.manipulation import unbind #DEFINE_ALIAS -from .tensor.manipulation import roll #DEFINE_ALIAS -from .tensor.manipulation import chunk #DEFINE_ALIAS -from .tensor.manipulation import tolist #DEFINE_ALIAS -from .tensor.math import abs #DEFINE_ALIAS -from .tensor.math import acos #DEFINE_ALIAS -from .tensor.math import asin #DEFINE_ALIAS -from .tensor.math import atan #DEFINE_ALIAS -from .tensor.math import ceil #DEFINE_ALIAS -from .tensor.math import cos #DEFINE_ALIAS -from .tensor.math import tan #DEFINE_ALIAS -from .tensor.math import cosh #DEFINE_ALIAS -from .tensor.math import cumsum #DEFINE_ALIAS -# from .tensor.math import elementwise_add #DEFINE_ALIAS -# from .tensor.math import elementwise_div #DEFINE_ALIAS -# from .tensor.math import elementwise_floordiv #DEFINE_ALIAS -# from .tensor.math import elementwise_mod #DEFINE_ALIAS -# from .tensor.math import elementwise_pow #DEFINE_ALIAS -# from .tensor.math import elementwise_sub #DEFINE_ALIAS -from .tensor.math import exp #DEFINE_ALIAS -from .tensor.math import floor #DEFINE_ALIAS -from .tensor.math import increment #DEFINE_ALIAS -from .tensor.math import log #DEFINE_ALIAS -from .tensor.math import log2 #DEFINE_ALIAS -from .tensor.math import log10 #DEFINE_ALIAS -from .tensor.math import multiplex #DEFINE_ALIAS -from .tensor.math import pow #DEFINE_ALIAS -from .tensor.math import reciprocal #DEFINE_ALIAS -# from .tensor.math import reduce_max #DEFINE_ALIAS -# from .tensor.math import reduce_min #DEFINE_ALIAS -# from .tensor.math import reduce_prod #DEFINE_ALIAS -# from .tensor.math import reduce_sum #DEFINE_ALIAS -from .tensor.math import all #DEFINE_ALIAS -from .tensor.math import any #DEFINE_ALIAS -from .tensor.math import round #DEFINE_ALIAS -from .tensor.math import rsqrt #DEFINE_ALIAS -from .tensor.math import scale #DEFINE_ALIAS -from .tensor.math import sign #DEFINE_ALIAS -from .tensor.math import sin #DEFINE_ALIAS -from .tensor.math import sinh #DEFINE_ALIAS -from .tensor.math import sqrt #DEFINE_ALIAS -from .tensor.math import square #DEFINE_ALIAS -from .tensor.math import stanh #DEFINE_ALIAS -from .tensor.math import sum #DEFINE_ALIAS -from .tensor.math import tanh #DEFINE_ALIAS -from .tensor.math import tanh_ #DEFINE_ALIAS -from .tensor.math import add_n #DEFINE_ALIAS -from .tensor.math import max #DEFINE_ALIAS -from .tensor.math import maximum #DEFINE_ALIAS -from .tensor.math import min #DEFINE_ALIAS -from .tensor.math import minimum #DEFINE_ALIAS -from .tensor.math import mm #DEFINE_ALIAS -from .tensor.math import divide #DEFINE_ALIAS -from .tensor.math import floor_divide #DEFINE_ALIAS -from .tensor.math import remainder #DEFINE_ALIAS -from .tensor.math import mod #DEFINE_ALIAS -from .tensor.math import floor_mod #DEFINE_ALIAS -from .tensor.math import multiply #DEFINE_ALIAS -from .tensor.math import add #DEFINE_ALIAS -from .tensor.math import subtract #DEFINE_ALIAS -from .tensor.math import atan #DEFINE_ALIAS -from .tensor.math import logsumexp #DEFINE_ALIAS -from .tensor.math import inverse #DEFINE_ALIAS -from .tensor.math import log1p #DEFINE_ALIAS -from .tensor.math import erf #DEFINE_ALIAS -from .tensor.math import addmm #DEFINE_ALIAS -from .tensor.math import clip #DEFINE_ALIAS -from .tensor.math import trace #DEFINE_ALIAS -from .tensor.math import kron #DEFINE_ALIAS -from .tensor.math import isfinite #DEFINE_ALIAS -from .tensor.math import isinf #DEFINE_ALIAS -from .tensor.math import isnan #DEFINE_ALIAS -from .tensor.math import prod #DEFINE_ALIAS -from .tensor.math import broadcast_shape #DEFINE_ALIAS -from .tensor.math import conj #DEFINE_ALIAS +from .tensor.attribute import rank # noqa: F401 +from .tensor.attribute import shape # noqa: F401 +from .tensor.attribute import real # noqa: F401 +from .tensor.attribute import imag # noqa: F401 +from .tensor.creation import to_tensor # noqa: F401 +from .tensor.creation import diag # noqa: F401 +from .tensor.creation import eye # noqa: F401 +from .tensor.creation import linspace # noqa: F401 +from .tensor.creation import ones # noqa: F401 +from .tensor.creation import ones_like # noqa: F401 +from .tensor.creation import zeros # noqa: F401 +from .tensor.creation import zeros_like # noqa: F401 +from .tensor.creation import arange # noqa: F401 +from .tensor.creation import full # noqa: F401 +from .tensor.creation import full_like # noqa: F401 +from .tensor.creation import triu # noqa: F401 +from .tensor.creation import tril # noqa: F401 +from .tensor.creation import meshgrid # noqa: F401 +from .tensor.creation import empty # noqa: F401 +from .tensor.creation import empty_like # noqa: F401 +from .tensor.creation import assign # noqa: F401 +from .tensor.linalg import matmul # noqa: F401 +from .tensor.linalg import dot # noqa: F401 +from .tensor.linalg import norm # noqa: F401 +from .tensor.linalg import transpose # noqa: F401 +from .tensor.linalg import dist # noqa: F401 +from .tensor.linalg import t # noqa: F401 +from .tensor.linalg import cross # noqa: F401 +from .tensor.linalg import cholesky # noqa: F401 +from .tensor.linalg import bmm # noqa: F401 +from .tensor.linalg import histogram # noqa: F401 +from .tensor.linalg import mv # noqa: F401 +from .tensor.logic import equal # noqa: F401 +from .tensor.logic import greater_equal # noqa: F401 +from .tensor.logic import greater_than # noqa: F401 +from .tensor.logic import is_empty # noqa: F401 +from .tensor.logic import less_equal # noqa: F401 +from .tensor.logic import less_than # noqa: F401 +from .tensor.logic import logical_and # noqa: F401 +from .tensor.logic import logical_not # noqa: F401 +from .tensor.logic import logical_or # noqa: F401 +from .tensor.logic import logical_xor # noqa: F401 +from .tensor.logic import not_equal # noqa: F401 +from .tensor.logic import allclose # noqa: F401 +from .tensor.logic import equal_all # noqa: F401 +from .tensor.logic import is_tensor # noqa: F401 +from .tensor.manipulation import cast # noqa: F401 +from .tensor.manipulation import concat # noqa: F401 +from .tensor.manipulation import expand # noqa: F401 +from .tensor.manipulation import broadcast_to # noqa: F401 +from .tensor.manipulation import expand_as # noqa: F401 +from .tensor.manipulation import tile # noqa: F401 +from .tensor.manipulation import flatten # noqa: F401 +from .tensor.manipulation import gather # noqa: F401 +from .tensor.manipulation import gather_nd # noqa: F401 +from .tensor.manipulation import reshape # noqa: F401 +from .tensor.manipulation import reshape_ # noqa: F401 +from .tensor.manipulation import flip as reverse # noqa: F401 +from .tensor.manipulation import scatter # noqa: F401 +from .tensor.manipulation import scatter_ # noqa: F401 +from .tensor.manipulation import scatter_nd_add # noqa: F401 +from .tensor.manipulation import scatter_nd # noqa: F401 +from .tensor.manipulation import shard_index # noqa: F401 +from .tensor.manipulation import slice # noqa: F401 +from .tensor.manipulation import split # noqa: F401 +from .tensor.manipulation import squeeze # noqa: F401 +from .tensor.manipulation import squeeze_ # noqa: F401 +from .tensor.manipulation import stack # noqa: F401 +from .tensor.manipulation import strided_slice # noqa: F401 +from .tensor.manipulation import transpose # noqa: F401 +from .tensor.manipulation import unique # noqa: F401 +from .tensor.manipulation import unsqueeze # noqa: F401 +from .tensor.manipulation import unsqueeze_ # noqa: F401 +from .tensor.manipulation import unstack # noqa: F401 +from .tensor.manipulation import flip # noqa: F401 +from .tensor.manipulation import unbind # noqa: F401 +from .tensor.manipulation import roll # noqa: F401 +from .tensor.manipulation import chunk # noqa: F401 +from .tensor.manipulation import tolist # noqa: F401 +from .tensor.math import abs # noqa: F401 +from .tensor.math import acos # noqa: F401 +from .tensor.math import asin # noqa: F401 +from .tensor.math import atan # noqa: F401 +from .tensor.math import ceil # noqa: F401 +from .tensor.math import cos # noqa: F401 +from .tensor.math import tan # noqa: F401 +from .tensor.math import cosh # noqa: F401 +from .tensor.math import cumsum # noqa: F401 +from .tensor.math import exp # noqa: F401 +from .tensor.math import floor # noqa: F401 +from .tensor.math import increment # noqa: F401 +from .tensor.math import log # noqa: F401 +from .tensor.math import log2 # noqa: F401 +from .tensor.math import log10 # noqa: F401 +from .tensor.math import multiplex # noqa: F401 +from .tensor.math import pow # noqa: F401 +from .tensor.math import reciprocal # noqa: F401 +from .tensor.math import all # noqa: F401 +from .tensor.math import any # noqa: F401 +from .tensor.math import round # noqa: F401 +from .tensor.math import rsqrt # noqa: F401 +from .tensor.math import scale # noqa: F401 +from .tensor.math import sign # noqa: F401 +from .tensor.math import sin # noqa: F401 +from .tensor.math import sinh # noqa: F401 +from .tensor.math import sqrt # noqa: F401 +from .tensor.math import square # noqa: F401 +from .tensor.math import stanh # noqa: F401 +from .tensor.math import sum # noqa: F401 +from .tensor.math import tanh # noqa: F401 +from .tensor.math import tanh_ # noqa: F401 +from .tensor.math import add_n # noqa: F401 +from .tensor.math import max # noqa: F401 +from .tensor.math import maximum # noqa: F401 +from .tensor.math import min # noqa: F401 +from .tensor.math import minimum # noqa: F401 +from .tensor.math import mm # noqa: F401 +from .tensor.math import divide # noqa: F401 +from .tensor.math import floor_divide # noqa: F401 +from .tensor.math import remainder # noqa: F401 +from .tensor.math import mod # noqa: F401 +from .tensor.math import floor_mod # noqa: F401 +from .tensor.math import multiply # noqa: F401 +from .tensor.math import add # noqa: F401 +from .tensor.math import subtract # noqa: F401 +from .tensor.math import atan # noqa: F401 +from .tensor.math import logsumexp # noqa: F401 +from .tensor.math import inverse # noqa: F401 +from .tensor.math import log1p # noqa: F401 +from .tensor.math import erf # noqa: F401 +from .tensor.math import addmm # noqa: F401 +from .tensor.math import clip # noqa: F401 +from .tensor.math import trace # noqa: F401 +from .tensor.math import kron # noqa: F401 +from .tensor.math import isfinite # noqa: F401 +from .tensor.math import isinf # noqa: F401 +from .tensor.math import isnan # noqa: F401 +from .tensor.math import prod # noqa: F401 +from .tensor.math import broadcast_shape # noqa: F401 +from .tensor.math import conj # noqa: F401 -from .tensor.random import multinomial #DEFINE_ALIAS -from .tensor.random import standard_normal -from .tensor.random import normal -from .tensor.random import uniform #DEFINE_ALIAS -from .tensor.random import randn #DEFINE_ALIAS -from .tensor.random import rand #DEFINE_ALIAS -from .tensor.random import randint #DEFINE_ALIAS -from .tensor.random import randperm #DEFINE_ALIAS -from .tensor.search import argmax #DEFINE_ALIAS -from .tensor.search import argmin #DEFINE_ALIAS -from .tensor.search import argsort #DEFINE_ALIAS -# from .tensor.search import has_inf #DEFINE_ALIAS -# from .tensor.search import has_nan #DEFINE_ALIAS -from .tensor.search import masked_select #DEFINE_ALIAS -from .tensor.search import topk #DEFINE_ALIAS -from .tensor.search import where #DEFINE_ALIAS -from .tensor.search import index_select #DEFINE_ALIAS -from .tensor.search import nonzero #DEFINE_ALIAS -from .tensor.search import sort #DEFINE_ALIAS +from .tensor.random import multinomial # noqa: F401 +from .tensor.random import standard_normal # noqa: F401 +from .tensor.random import normal # noqa: F401 +from .tensor.random import uniform # noqa: F401 +from .tensor.random import randn # noqa: F401 +from .tensor.random import rand # noqa: F401 +from .tensor.random import randint # noqa: F401 +from .tensor.random import randperm # noqa: F401 +from .tensor.search import argmax # noqa: F401 +from .tensor.search import argmin # noqa: F401 +from .tensor.search import argsort # noqa: F401 +from .tensor.search import masked_select # noqa: F401 +from .tensor.search import topk # noqa: F401 +from .tensor.search import where # noqa: F401 +from .tensor.search import index_select # noqa: F401 +from .tensor.search import nonzero # noqa: F401 +from .tensor.search import sort # noqa: F401 -from .tensor.to_string import set_printoptions #DEFINE_ALIAS +from .tensor.to_string import set_printoptions # noqa: F401 -from .framework.random import seed #DEFINE_ALIAS -from .framework.random import get_cuda_rng_state #DEFINE_ALIAS -from .framework.random import set_cuda_rng_state #DEFINE_ALIAS -from .framework import ParamAttr #DEFINE_ALIAS -# from .framework import create_global_var #DEFINE_ALIAS -from .framework import create_parameter #DEFINE_ALIAS -from .framework import CPUPlace #DEFINE_ALIAS -from .framework import CUDAPlace #DEFINE_ALIAS -from .framework import NPUPlace #DEFINE_ALIAS -from .framework import CUDAPinnedPlace #DEFINE_ALIAS +from .framework.random import seed # noqa: F401 +from .framework.random import get_cuda_rng_state # noqa: F401 +from .framework.random import set_cuda_rng_state # noqa: F401 +from .framework import ParamAttr # noqa: F401 +from .framework import create_parameter # noqa: F401 +from .framework import CPUPlace # noqa: F401 +from .framework import CUDAPlace # noqa: F401 +from .framework import NPUPlace # noqa: F401 +from .framework import CUDAPinnedPlace # noqa: F401 -from .framework import grad #DEFINE_ALIAS -from .framework import no_grad #DEFINE_ALIAS -from .framework import set_grad_enabled #DEFINE_ALIAS -from .framework import save #DEFINE_ALIAS -from .framework import load #DEFINE_ALIAS -from .framework import DataParallel #DEFINE_ALIAS +from .framework import grad # noqa: F401 +from .framework import no_grad # noqa: F401 +from .framework import set_grad_enabled # noqa: F401 +from .framework import save # noqa: F401 +from .framework import load # noqa: F401 +from .framework import DataParallel # noqa: F401 from .framework import set_default_dtype #DEFINE_ALIAS from .framework import get_default_dtype #DEFINE_ALIAS from .framework import set_grad_enabled #DEFINE_ALIAS -from .tensor.search import index_sample #DEFINE_ALIAS -from .tensor.stat import mean #DEFINE_ALIAS -# from .tensor.stat import reduce_mean #DEFINE_ALIAS -from .tensor.stat import std #DEFINE_ALIAS -from .tensor.stat import var #DEFINE_ALIAS -# from .fluid.data import data -from .tensor.stat import numel #DEFINE_ALIAS -from .tensor.stat import median #DEFINE_ALIAS -from .device import get_cudnn_version -from .device import set_device -from .device import get_device -from .device import is_compiled_with_cuda #DEFINE_ALIAS -from .device import is_compiled_with_xpu -from .device import is_compiled_with_npu -from .device import XPUPlace -# from .tensor.tensor import Tensor #DEFINE_ALIAS -# from .tensor.tensor import LoDTensor #DEFINE_ALIAS -# from .tensor.tensor import LoDTensorArray #DEFINE_ALIAS +from .tensor.search import index_sample # noqa: F401 +from .tensor.stat import mean # noqa: F401 +from .tensor.stat import std # noqa: F401 +from .tensor.stat import var # noqa: F401 +from .tensor.stat import numel # noqa: F401 +from .tensor.stat import median # noqa: F401 +from .device import get_cudnn_version # noqa: F401 +from .device import set_device # noqa: F401 +from .device import get_device # noqa: F401 +from .fluid.framework import is_compiled_with_cuda # noqa: F401 +from .device import is_compiled_with_xpu # noqa: F401 +from .device import is_compiled_with_npu # noqa: F401 +from .device import XPUPlace # noqa: F401 -from .fluid.dygraph.base import enable_dygraph as disable_static #DEFINE_ALIAS -from .fluid.dygraph.base import disable_dygraph as enable_static #DEFINE_ALIAS -from .fluid.framework import in_dygraph_mode as in_dynamic_mode #DEFINE_ALIAS -from .fluid.layers import crop_tensor as crop #DEFINE_ALIAS - -from . import jit -from . import static -from . import amp -from . import onnx +from .fluid.dygraph.base import enable_dygraph as disable_static # noqa: F401 +from .fluid.dygraph.base import disable_dygraph as enable_static # noqa: F401 +from .fluid.framework import in_dygraph_mode as in_dynamic_mode # noqa: F401 +from .fluid.layers import crop_tensor as crop # noqa: F401 # high-level api -from .hapi import Model -from .hapi import callbacks -from .hapi import summary -from .hapi import flops -from .hapi import hub +from .hapi import Model # noqa: F401 +from .hapi import callbacks # noqa: F401 +from .hapi import summary # noqa: F401 +from .hapi import flops # noqa: F401 +from .hapi import hub # noqa: F401 -import paddle.text -import paddle.vision +import paddle.text # noqa: F401 +import paddle.vision # noqa: F401 +from .tensor.random import check_shape # noqa: F401 disable_static() + +__all__ = [ #noqa + 'dtype', + 'uint8', + 'int8', + 'int16', + 'int32', + 'int64', + 'float16', + 'float32', + 'float64', + 'bfloat16', + 'bool', + 'complex64', + 'complex128', + 'addmm', + 'allclose', + 't', + 'add', + 'subtract', + 'diag', + 'isnan', + 'scatter_nd_add', + 'unstack', + 'get_default_dtype', + 'save', + 'multinomial', + 'get_cuda_rng_state', + 'rank', + 'empty_like', + 'eye', + 'cumsum', + 'sign', + 'is_empty', + 'equal', + 'equal_all', + 'is_tensor', + 'cross', + 'where', + 'log1p', + 'cos', + 'tan', + 'mean', + 'XPUPlace', + 'mv', + 'in_dynamic_mode', + 'min', + 'any', + 'slice', + 'normal', + 'logsumexp', + 'full', + 'unsqueeze', + 'unsqueeze_', + 'argmax', + 'Model', + 'callbacks', + 'summary', + 'flops', + 'hub', + 'sort', + 'split', + 'logical_and', + 'full_like', + 'less_than', + 'kron', + 'clip', + 'Tensor', + 'crop', + 'ParamAttr', + 'stanh', + 'randint', + 'assign', + 'gather', + 'scale', + 'zeros', + 'rsqrt', + 'squeeze', + 'squeeze_', + 'to_tensor', + 'gather_nd', + 'isinf', + 'set_device', + 'uniform', + 'floor_divide', + 'remainder', + 'floor_mod', + 'roll', + 'batch', + 'max', + 'norm', + 'logical_or', + 'mm', + 'flip', + 'histogram', + 'multiplex', + 'CUDAPlace', + 'NPUPlace', + 'empty', + 'shape', + 'real', + 'imag', + 'reciprocal', + 'rand', + 'less_equal', + 'triu', + 'is_compiled_with_cuda', + 'sin', + 'dist', + 'unbind', + 'meshgrid', + 'arange', + 'load', + 'numel', + 'median', + 'inverse', + 'no_grad', + 'set_grad_enabled', + 'mod', + 'abs', + 'tril', + 'pow', + 'zeros_like', + 'maximum', + 'topk', + 'index_select', + 'CPUPlace', + 'matmul', + 'seed', + 'acos', + 'logical_xor', + 'exp', + 'bernoulli', + 'summary', + 'sinh', + 'is_compiled_with_xpu', + 'is_compiled_with_npu', + 'round', + 'DataParallel', + 'argmin', + 'prod', + 'broadcast_shape', + 'conj', + 'square', + 'divide', + 'ceil', + 'atan', + 'expand', + 'broadcast_to', + 'ones_like', + 'index_sample', + 'cast', + 'grad', + 'all', + 'ones', + 'not_equal', + 'sum', + 'tile', + 'get_device', + 'greater_equal', + 'isfinite', + 'create_parameter', + 'dot', + 'increment', + 'erf', + 'bmm', + 'chunk', + 'tolist', + 'greater_than', + 'shard_index', + 'argsort', + 'tanh', + 'tanh_', + 'transpose', + 'randn', + 'strided_slice', + 'unique', + 'set_cuda_rng_state', + 'set_printoptions', + 'std', + 'flatten', + 'asin', + 'multiply', + 'disable_static', + 'masked_select', + 'var', + 'trace', + 'enable_static', + 'scatter_nd', + 'set_default_dtype', + 'expand_as', + 'get_cudnn_version', + 'stack', + 'sqrt', + 'cholesky', + 'randperm', + 'linspace', + 'reshape', + 'reshape_', + 'reverse', + 'nonzero', + 'CUDAPinnedPlace', + 'logical_not', + 'add_n', + 'minimum', + 'ComplexTensor', + 'scatter', + 'scatter_', + 'floor', + 'cosh', + 'log', + 'log2', + 'log10', + 'concat', + 'check_shape' +] From 33703da8eecedfcf61f3548d1b6b5d434dce13c6 Mon Sep 17 00:00:00 2001 From: jiangcheng Date: Wed, 28 Apr 2021 16:31:19 +0800 Subject: [PATCH 004/156] [Cherry-pick] Optimize update_loss_scaling_op(#32554) (#32606) * optimize update_loss_scaling_op by fused for loop to one kernel, test=develop * remove useless while loop and optimize variable name, test=develop * optimize variable name from out_addrs_tensor to out_addrs_mem, test=develop * optimize variable name for readable by change prefix identifier from t_ to local_ --- .../amp/check_finite_and_unscale_op.cu | 63 +++++++------ .../operators/amp/update_loss_scaling_op.cu | 93 ++++++++++++++++--- 2 files changed, 113 insertions(+), 43 deletions(-) diff --git a/paddle/fluid/operators/amp/check_finite_and_unscale_op.cu b/paddle/fluid/operators/amp/check_finite_and_unscale_op.cu index 2c3a9c366e4fd..c699486a9140a 100644 --- a/paddle/fluid/operators/amp/check_finite_and_unscale_op.cu +++ b/paddle/fluid/operators/amp/check_finite_and_unscale_op.cu @@ -39,33 +39,36 @@ __global__ void CheckFiniteAndUnscale(const T** xs, const MT* scale, __syncthreads(); const int64_t num = s_starts[size]; - int pre_xs_index = 0; - bool t_found_inf = false; - const MT t_scale = *scale; + int xs_index = 0; + bool local_found_inf = false; + const MT local_scale = *scale; for (int64_t idx = tid; idx < num; idx += gridDim.x * blockDim.x) { - // get the xs's index of thread - int xs_index = pre_xs_index; - while (idx < s_starts[xs_index]) xs_index++; - // avoid some tensor's numel is zero - while (idx >= s_starts[xs_index]) xs_index++; - pre_xs_index = xs_index - 1; + // get the "out" index of "id" + // For example: + // idx = 15, starts = [0, 10, 10, 20, 30] + // because 10 <= idx < 20 ==> + // the idx element locate in the 3rd tensor (notice the 2nd tensor size is + // 0) + int next_xs_index = xs_index; + while (idx >= s_starts[next_xs_index]) next_xs_index++; + xs_index = next_xs_index - 1; // get in data and out data - const T* in = xs[pre_xs_index]; - T* out = outs[pre_xs_index]; - int64_t in_idx = idx - s_starts[pre_xs_index]; + const T* in = xs[xs_index]; + T* out = outs[xs_index]; + int64_t in_idx = idx - s_starts[xs_index]; // Unscale - MT val = static_cast(in[in_idx]) * t_scale; + MT val = static_cast(in[in_idx]) * local_scale; T narrow_val = static_cast(val); out[in_idx] = narrow_val; // CheckFinite if (!isfinite(narrow_val)) { - t_found_inf = true; + local_found_inf = true; } } - if (t_found_inf) { + if (local_found_inf) { *found_inf = true; } } @@ -94,28 +97,30 @@ class CheckFiniteAndUnscaleGpuKernel : public framework::OpKernel { scale_data, inverse_scale_v, found_inf_data); size_t xs_size = xs.size(); + const auto& cpu_place = platform::CPUPlace(); // calculate each tensor's start index and copy to device auto h_starts_tensor = - memory::Alloc(platform::CPUPlace(), (xs_size + 1) * sizeof(int64_t)); + memory::Alloc(cpu_place, (xs_size + 1) * sizeof(int64_t)); int64_t* h_starts = reinterpret_cast(h_starts_tensor->ptr()); auto d_starts_tensor = memory::Alloc(dev_ctx, (xs_size + 1) * sizeof(int64_t)); int64_t* d_starts = reinterpret_cast(d_starts_tensor->ptr()); + // the start index value of each tensor is + // the sum of previous tensor's size. For example: + // xs = [10, 0, 10, 10] ==> starts = [0, 10, 10, 20, 30] h_starts[0] = 0; for (int i = 1; i <= xs_size; i++) { - // the start index value of each tensor is - // the sum of previous tensor's size h_starts[i] = h_starts[i - 1] + xs[i - 1]->numel(); } int64_t total_num = h_starts[xs_size]; memory::Copy(BOOST_GET_CONST(platform::CUDAPlace, dev_ctx.GetPlace()), - d_starts, platform::CPUPlace(), h_starts, - (xs_size + 1) * sizeof(int64_t), dev_ctx.stream()); + d_starts, cpu_place, h_starts, (xs_size + 1) * sizeof(int64_t), + dev_ctx.stream()); // copy each tensor's data address to device - auto h_mem = memory::Alloc(platform::CPUPlace(), 2 * xs_size * sizeof(T*)); + auto h_mem = memory::Alloc(cpu_place, 2 * xs_size * sizeof(T*)); const T** h_xs = reinterpret_cast(h_mem->ptr()); T** h_outs = reinterpret_cast(h_mem->ptr()) + xs_size; @@ -128,16 +133,18 @@ class CheckFiniteAndUnscaleGpuKernel : public framework::OpKernel { h_outs[i] = outs[i]->mutable_data(dev_ctx.GetPlace()); } memory::Copy(BOOST_GET_CONST(platform::CUDAPlace, dev_ctx.GetPlace()), d_xs, - platform::CPUPlace(), h_xs, 2 * xs_size * sizeof(T*), - dev_ctx.stream()); + cpu_place, h_xs, 2 * xs_size * sizeof(T*), dev_ctx.stream()); // Launch Kernel - int block = 1024; - int block_num = block * 20; // each thread deal with 20 number - int grid = (total_num + block_num - 1) / block_num; + int threads_per_block = std::min(static_cast(1024), total_num); + int elements_per_block = + threads_per_block * 20; // each thread deal with 20 number + int blocks_per_grid = + (total_num + elements_per_block - 1) / elements_per_block; VLOG(3) << "launch kernel"; - CheckFiniteAndUnscale<<< - grid, block, (xs_size + 1) * sizeof(int64_t), dev_ctx.stream()>>>( + CheckFiniteAndUnscale< + T, MPDType><<>>( d_xs, inverse_scale_v, xs_size, d_starts, found_inf_data, d_outs); VLOG(3) << "finish kernel"; } diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op.cu b/paddle/fluid/operators/amp/update_loss_scaling_op.cu index b48b0e7889293..de1f83c1ee50d 100644 --- a/paddle/fluid/operators/amp/update_loss_scaling_op.cu +++ b/paddle/fluid/operators/amp/update_loss_scaling_op.cu @@ -34,13 +34,39 @@ __global__ void GpuUpdateLossScaling( } template -__global__ void FillIf(T* data, const int64_t num, const T value, - const bool* has_inf) { - if (*has_inf) { - int tid = threadIdx.x + blockIdx.x * blockDim.x; - for (int i = tid; i < num; i += blockDim.x * gridDim.x) { - data[i] = value; - } +__global__ void FusedFillIf(T** outs, const size_t xs_size, + const int64_t* starts, const T value, + const bool* has_inf) { + if (!(*has_inf)) return; + + const int tid = threadIdx.x + blockIdx.x * blockDim.x; + + // copy starts array from global memory to shared memory + extern __shared__ int64_t s_starts[]; + for (int i = threadIdx.x; i <= xs_size; i += blockDim.x) { + s_starts[i] = starts[i]; + } + __syncthreads(); + + const int64_t total_num = s_starts[xs_size]; + int out_index = 0; + + for (int64_t id = tid; id < total_num; id += blockDim.x * gridDim.x) { + // get the "out" index of "id" + // For example: + // id = 15, starts = [0, 10, 10, 20, 30] + // because 10 <= id < 20 ==> + // the id element locate in the 3rd tensor (notice the 2nd tensor size is 0) + int next_out_index = out_index; + while (id >= s_starts[next_out_index]) next_out_index++; + out_index = next_out_index - 1; + + // get data pointer and index + T* out_data = outs[out_index]; + int64_t idx = id - s_starts[out_index]; + + // set value + out_data[idx] = value; } } @@ -68,15 +94,52 @@ class LazyZeros { const bool* found_inf_data, const std::vector& xs, const std::vector& outs) const { - for (size_t i = 0; i < xs.size(); ++i) { - auto* out = outs[i]; - T* out_data = out->mutable_data(dev_ctx.GetPlace()); - int64_t num = out->numel(); - int block = 1024; - int grid = (block - 1 + num) / block; - FillIf<<>>( - out_data, num, static_cast(0), found_inf_data); + size_t xs_size = xs.size(); + const auto& cpu_place = platform::CPUPlace(); + // alloc each tensor's start index and copy to device + auto h_in_starts_mem = + memory::Alloc(cpu_place, (xs_size + 1) * sizeof(int64_t)); + int64_t* h_starts = reinterpret_cast(h_in_starts_mem->ptr()); + + auto d_in_starts_mem = + memory::Alloc(dev_ctx, (xs_size + 1) * sizeof(int64_t)); + int64_t* d_starts = reinterpret_cast(d_in_starts_mem->ptr()); + + // the start index value of each tensor is + // the sum of previous tensor's size. For example: + // outs = [10, 0, 10, 10] ==> starts = [0, 10, 10, 20, 30] + h_starts[0] = 0; + for (int i = 0; i < xs_size; i++) { + h_starts[i + 1] = h_starts[i] + outs[i]->numel(); } + memory::Copy(BOOST_GET_CONST(platform::CUDAPlace, dev_ctx.GetPlace()), + d_starts, cpu_place, h_starts, (xs_size + 1) * sizeof(int64_t), + dev_ctx.stream()); + + // copy each tensor of "outs" data address array to device + auto h_out_addrs_mem = memory::Alloc(cpu_place, xs_size * sizeof(T*)); + T** h_out_addrs = reinterpret_cast(h_out_addrs_mem->ptr()); + + auto d_out_addrs_mem = memory::Alloc(dev_ctx, xs_size * sizeof(T*)); + T** d_out_addrs = reinterpret_cast(d_out_addrs_mem->ptr()); + + for (size_t i = 0; i < xs_size; ++i) { + h_out_addrs[i] = outs[i]->mutable_data(dev_ctx.GetPlace()); + } + memory::Copy(BOOST_GET_CONST(platform::CUDAPlace, dev_ctx.GetPlace()), + d_out_addrs, cpu_place, h_out_addrs, xs_size * sizeof(T*), + dev_ctx.stream()); + + // launch cuda kernel + int64_t total_num = h_starts[xs_size]; + int64_t threads_per_block = std::min(static_cast(1024), total_num); + int64_t elements_per_block = + threads_per_block * 50; // each thread deal with 50 data + int64_t blocks_per_grid = + (total_num + elements_per_block - 1) / elements_per_block; + FusedFillIf<<>>( + d_out_addrs, xs_size, d_starts, static_cast(0), found_inf_data); } }; From 056a2fca695b6b5ed9fb278aead51942b9ae3bfb Mon Sep 17 00:00:00 2001 From: wenbin Date: Wed, 28 Apr 2021 22:25:09 +0800 Subject: [PATCH 005/156] conservative judgment (#32619) --- paddle/fluid/inference/tensorrt/convert/elementwise_op.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc index 5419933e40736..19d79510547ec 100644 --- a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc @@ -25,6 +25,10 @@ static bool CheckDims(const nvinfer1::Dims& dims_x, return false; } for (int i = 0; i < dims_x.nbDims; i++) { + // conservative judgment + if (dims_x.d[i] == -1 || dims_y.d[i] == -1) { + return false; + } if (dims_x.d[i] != dims_y.d[i]) { return false; } From e60c08f78503c326f52518954f55cf22769fb9e7 Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Thu, 29 Apr 2021 08:40:00 +0800 Subject: [PATCH 006/156] add __all__=[] to python files not in API public list; import * only support in API public list files (#32644) --- python/paddle/dataset/cifar.py | 2 ++ python/paddle/dataset/common.py | 2 ++ python/paddle/dataset/conll05.py | 2 ++ python/paddle/dataset/flowers.py | 2 ++ python/paddle/dataset/image.py | 2 ++ python/paddle/dataset/imdb.py | 2 ++ python/paddle/dataset/imikolov.py | 2 ++ python/paddle/dataset/mnist.py | 2 ++ python/paddle/dataset/movielens.py | 2 ++ python/paddle/dataset/tests/cifar_test.py | 2 ++ python/paddle/dataset/tests/flowers_test.py | 2 ++ python/paddle/dataset/tests/imdb_test.py | 2 ++ python/paddle/dataset/tests/imikolov_test.py | 2 ++ python/paddle/dataset/tests/mnist_test.py | 2 ++ python/paddle/dataset/tests/test_image.py | 2 ++ python/paddle/dataset/tests/voc2012_test.py | 2 ++ python/paddle/dataset/tests/wmt16_test.py | 2 ++ python/paddle/dataset/uci_housing.py | 2 ++ python/paddle/dataset/voc2012.py | 3 ++- python/paddle/dataset/wmt14.py | 2 ++ python/paddle/dataset/wmt16.py | 2 ++ python/paddle/framework/__init__.py | 2 ++ python/paddle/framework/dtype.py | 7 ++----- python/paddle/framework/framework.py | 2 ++ python/paddle/framework/io.py | 2 ++ python/paddle/framework/random.py | 2 ++ python/paddle/nn/clip.py | 2 ++ python/paddle/nn/decode.py | 2 ++ python/paddle/nn/functional/activation.py | 2 ++ python/paddle/nn/functional/common.py | 2 ++ python/paddle/nn/functional/conv.py | 2 ++ python/paddle/nn/functional/extension.py | 2 ++ python/paddle/nn/functional/input.py | 2 ++ python/paddle/nn/functional/loss.py | 2 ++ python/paddle/nn/functional/norm.py | 2 ++ python/paddle/nn/functional/pooling.py | 2 ++ python/paddle/nn/functional/vision.py | 2 ++ python/paddle/nn/initializer/assign.py | 2 ++ python/paddle/nn/initializer/constant.py | 2 ++ python/paddle/nn/initializer/kaiming.py | 2 ++ python/paddle/nn/initializer/normal.py | 2 ++ python/paddle/nn/initializer/uniform.py | 2 ++ python/paddle/nn/initializer/xavier.py | 2 ++ python/paddle/nn/layer/__init__.py | 2 ++ python/paddle/nn/layer/activation.py | 2 ++ python/paddle/nn/layer/common.py | 2 ++ python/paddle/nn/layer/container.py | 2 +- python/paddle/nn/layer/conv.py | 2 ++ python/paddle/nn/layer/distance.py | 2 ++ python/paddle/nn/layer/loss.py | 2 ++ python/paddle/nn/layer/norm.py | 2 ++ python/paddle/nn/layer/pooling.py | 2 ++ python/paddle/nn/layer/rnn.py | 2 ++ python/paddle/nn/layer/transformer.py | 2 ++ python/paddle/nn/layer/vision.py | 2 ++ python/paddle/nn/utils/weight_norm_hook.py | 2 ++ python/paddle/optimizer/adadelta.py | 2 ++ python/paddle/optimizer/adagrad.py | 2 ++ python/paddle/optimizer/adam.py | 2 ++ python/paddle/optimizer/adamax.py | 2 ++ python/paddle/optimizer/adamw.py | 2 ++ python/paddle/optimizer/lamb.py | 2 ++ python/paddle/optimizer/momentum.py | 2 ++ python/paddle/optimizer/optimizer.py | 2 ++ python/paddle/optimizer/rmsprop.py | 2 ++ python/paddle/optimizer/sgd.py | 2 ++ python/paddle/proto/__init__.py | 2 ++ python/paddle/reader/decorator.py | 2 ++ python/paddle/reader/tests/decorator_test.py | 2 ++ python/paddle/static/amp/__init__.py | 2 ++ python/paddle/static/input.py | 2 ++ python/paddle/static/io.py | 2 ++ python/paddle/static/nn/common.py | 2 ++ python/paddle/tensor/array.py | 2 ++ python/paddle/tensor/attribute.py | 2 ++ python/paddle/tensor/creation.py | 2 ++ python/paddle/tensor/linalg.py | 2 ++ python/paddle/tensor/logic.py | 2 ++ python/paddle/tensor/manipulation.py | 2 ++ python/paddle/tensor/math.py | 2 ++ python/paddle/tensor/random.py | 2 ++ python/paddle/tensor/search.py | 2 ++ python/paddle/tensor/stat.py | 2 ++ python/paddle/tensor/to_string.py | 2 ++ python/paddle/tests/test_dataset_cifar.py | 2 +- python/paddle/tests/test_dataset_conll05.py | 2 +- python/paddle/tests/test_dataset_imdb.py | 2 +- python/paddle/tests/test_dataset_imikolov.py | 2 +- python/paddle/tests/test_dataset_movielens.py | 2 +- python/paddle/tests/test_dataset_uci_housing.py | 2 +- python/paddle/tests/test_dataset_wmt.py | 2 +- python/paddle/tests/test_datasets.py | 2 +- python/paddle/text/datasets/__init__.py | 2 ++ python/paddle/text/datasets/conll05.py | 2 ++ python/paddle/text/datasets/imdb.py | 2 ++ python/paddle/text/datasets/imikolov.py | 2 ++ python/paddle/text/datasets/movielens.py | 2 ++ python/paddle/text/datasets/uci_housing.py | 2 ++ python/paddle/text/datasets/wmt14.py | 2 ++ python/paddle/text/datasets/wmt16.py | 2 ++ python/paddle/utils/deprecated.py | 2 ++ python/paddle/utils/download.py | 2 ++ python/paddle/utils/image_util.py | 2 ++ python/paddle/utils/install_check.py | 2 ++ python/paddle/utils/lazy_import.py | 2 ++ python/paddle/utils/op_version.py | 2 ++ 106 files changed, 203 insertions(+), 15 deletions(-) diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index a6b6e28c0f5a3..e3d239e2cdf45 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -37,6 +37,8 @@ import six from six.moves import cPickle as pickle +__all__ = [] + URL_PREFIX = 'https://dataset.bj.bcebos.com/cifar/' CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz' CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a' diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index cff0c6257387c..2a476f63862cf 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -26,6 +26,8 @@ import six.moves.cPickle as pickle import glob +__all__ = [] + HOME = os.path.expanduser('~') DATA_HOME = os.path.join(HOME, '.cache', 'paddle', 'dataset') diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index 96fd5ae7d76c8..65cf04f05b7f0 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -30,6 +30,8 @@ import paddle.utils.deprecated as deprecated from six.moves import zip, range +__all__ = [] + DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz' DATA_MD5 = '387719152ae52d60422c016e92a742fc' WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt' diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 67ffd8e1ee1ed..3b437a1f07440 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -51,6 +51,8 @@ from six.moves import cPickle as pickle from paddle.utils import try_import +__all__ = [] + DATA_URL = 'http://paddlemodels.bj.bcebos.com/flowers/102flowers.tgz' LABEL_URL = 'http://paddlemodels.bj.bcebos.com/flowers/imagelabels.mat' SETID_URL = 'http://paddlemodels.bj.bcebos.com/flowers/setid.mat' diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 31329cd978cb5..c20672c2ce157 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -58,6 +58,8 @@ import tarfile import six.moves.cPickle as pickle +__all__ = [] + def _check_cv2(): if cv2 is None: diff --git a/python/paddle/dataset/imdb.py b/python/paddle/dataset/imdb.py index 33ae4405c502b..9a6c8e837ed46 100644 --- a/python/paddle/dataset/imdb.py +++ b/python/paddle/dataset/imdb.py @@ -30,6 +30,8 @@ import string import six +__all__ = [] + #URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz' URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz' MD5 = '7c2ac02c03563afcf9b574c7e56c153a' diff --git a/python/paddle/dataset/imikolov.py b/python/paddle/dataset/imikolov.py index 3b8b12303c949..7a4efe27aa961 100644 --- a/python/paddle/dataset/imikolov.py +++ b/python/paddle/dataset/imikolov.py @@ -27,6 +27,8 @@ import tarfile import six +__all__ = [] + #URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz' MD5 = '30177ea32e27c525793142b6bf2c8e2d' diff --git a/python/paddle/dataset/mnist.py b/python/paddle/dataset/mnist.py index 06e8174a61e80..e4f724bd66d13 100644 --- a/python/paddle/dataset/mnist.py +++ b/python/paddle/dataset/mnist.py @@ -27,6 +27,8 @@ import struct from six.moves import range +__all__ = [] + URL_PREFIX = 'https://dataset.bj.bcebos.com/mnist/' TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' TEST_IMAGE_MD5 = '9fb629c4189551a2d022fa330f9573f3' diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index 23781b65785b1..862ac586bc964 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -34,6 +34,8 @@ import six import paddle.compat as cpt +__all__ = [] + age_table = [1, 18, 25, 35, 45, 50, 56] #URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip' diff --git a/python/paddle/dataset/tests/cifar_test.py b/python/paddle/dataset/tests/cifar_test.py index 8e514f0fd9a18..54dff6b40cf3c 100644 --- a/python/paddle/dataset/tests/cifar_test.py +++ b/python/paddle/dataset/tests/cifar_test.py @@ -17,6 +17,8 @@ import paddle.dataset.cifar import unittest +__all__ = [] + class TestCIFAR(unittest.TestCase): def check_reader(self, reader): diff --git a/python/paddle/dataset/tests/flowers_test.py b/python/paddle/dataset/tests/flowers_test.py index 06a0a7761cfa1..256c116b7cff6 100644 --- a/python/paddle/dataset/tests/flowers_test.py +++ b/python/paddle/dataset/tests/flowers_test.py @@ -17,6 +17,8 @@ import paddle.dataset.flowers import unittest +__all__ = [] + class TestFlowers(unittest.TestCase): def check_reader(self, reader): diff --git a/python/paddle/dataset/tests/imdb_test.py b/python/paddle/dataset/tests/imdb_test.py index 613c5f8edb289..264b0f232fa80 100644 --- a/python/paddle/dataset/tests/imdb_test.py +++ b/python/paddle/dataset/tests/imdb_test.py @@ -18,6 +18,8 @@ import unittest import re +__all__ = [] + TRAIN_POS_PATTERN = re.compile(r"aclImdb/train/pos/.*\.txt$") TRAIN_NEG_PATTERN = re.compile(r"aclImdb/train/neg/.*\.txt$") TRAIN_PATTERN = re.compile(r"aclImdb/train/.*\.txt$") diff --git a/python/paddle/dataset/tests/imikolov_test.py b/python/paddle/dataset/tests/imikolov_test.py index 1f78a5dd4d1a0..5556274211fc3 100644 --- a/python/paddle/dataset/tests/imikolov_test.py +++ b/python/paddle/dataset/tests/imikolov_test.py @@ -19,6 +19,8 @@ WORD_DICT = paddle.dataset.imikolov.build_dict() +__all__ = [] + class TestMikolov(unittest.TestCase): def check_reader(self, reader, n): diff --git a/python/paddle/dataset/tests/mnist_test.py b/python/paddle/dataset/tests/mnist_test.py index fbb5d926494e3..238b58244e147 100644 --- a/python/paddle/dataset/tests/mnist_test.py +++ b/python/paddle/dataset/tests/mnist_test.py @@ -17,6 +17,8 @@ import paddle.dataset.mnist import unittest +__all__ = [] + class TestMNIST(unittest.TestCase): def check_reader(self, reader): diff --git a/python/paddle/dataset/tests/test_image.py b/python/paddle/dataset/tests/test_image.py index 32d2eb17ae673..259939d62f641 100644 --- a/python/paddle/dataset/tests/test_image.py +++ b/python/paddle/dataset/tests/test_image.py @@ -19,6 +19,8 @@ import paddle.dataset.image as image +__all__ = [] + class Image(unittest.TestCase): def test_resize_flip_chw(self): diff --git a/python/paddle/dataset/tests/voc2012_test.py b/python/paddle/dataset/tests/voc2012_test.py index cddeb91cab2c0..21c24e6df823f 100644 --- a/python/paddle/dataset/tests/voc2012_test.py +++ b/python/paddle/dataset/tests/voc2012_test.py @@ -17,6 +17,8 @@ import paddle.dataset.voc2012 import unittest +__all__ = [] + class TestVOC(unittest.TestCase): def check_reader(self, reader): diff --git a/python/paddle/dataset/tests/wmt16_test.py b/python/paddle/dataset/tests/wmt16_test.py index be121bb101219..68a9819c8f335 100644 --- a/python/paddle/dataset/tests/wmt16_test.py +++ b/python/paddle/dataset/tests/wmt16_test.py @@ -17,6 +17,8 @@ import paddle.dataset.wmt16 import unittest +__all__ = [] + class TestWMT16(unittest.TestCase): def checkout_one_sample(self, sample): diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 1bc2098350f53..0ac65f0fda46b 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -29,6 +29,8 @@ import paddle.dataset.common import paddle.utils.deprecated as deprecated +__all__ = [] + URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data' MD5 = 'd4accdce7a25600298819f8e28e8d593' feature_names = [ diff --git a/python/paddle/dataset/voc2012.py b/python/paddle/dataset/voc2012.py index 1575b44cd1677..5784e739b418e 100644 --- a/python/paddle/dataset/voc2012.py +++ b/python/paddle/dataset/voc2012.py @@ -25,10 +25,11 @@ import io import numpy as np from paddle.dataset.common import download -from paddle.dataset.image import * import paddle.utils.deprecated as deprecated from PIL import Image +__all__ = [] + VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/\ VOCtrainval_11-May-2012.tar' diff --git a/python/paddle/dataset/wmt14.py b/python/paddle/dataset/wmt14.py index 818f4b28ba143..c842ceaa09133 100644 --- a/python/paddle/dataset/wmt14.py +++ b/python/paddle/dataset/wmt14.py @@ -30,6 +30,8 @@ import paddle.compat as cpt import paddle.utils.deprecated as deprecated +__all__ = [] + URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' 'cslm_joint_paper/data/dev+test.tgz') MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index 6804e7ab5fc33..320ef139f7700 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -40,6 +40,8 @@ import paddle.compat as cpt import paddle.utils.deprecated as deprecated +__all__ = [] + DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz") DATA_MD5 = "0c38be43600334966403524a40dcd81e" diff --git a/python/paddle/framework/__init__.py b/python/paddle/framework/__init__.py index 660267c24e57e..ce84fb739c000 100644 --- a/python/paddle/framework/__init__.py +++ b/python/paddle/framework/__init__.py @@ -34,3 +34,5 @@ from .io import save # noqa: F401 from .io import load # noqa: F401 from ..fluid.dygraph.parallel import DataParallel # noqa: F401 + +__all__ = [] diff --git a/python/paddle/framework/dtype.py b/python/paddle/framework/dtype.py index 3eeaa6e74eceb..f49f748975882 100644 --- a/python/paddle/framework/dtype.py +++ b/python/paddle/framework/dtype.py @@ -12,11 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = [ - "dtype", "uint8", "int8", "int16", "int32", "int64", "bfloat16", "float16", - "float32", "float64", "complex64", "complex128", "bool" -] - from ..fluid.core import VarDesc dtype = VarDesc.VarType @@ -38,3 +33,5 @@ complex128 = VarDesc.VarType.COMPLEX128 bool = VarDesc.VarType.BOOL + +__all__ = [] diff --git a/python/paddle/framework/framework.py b/python/paddle/framework/framework.py index f50285010cc5d..17eaa82cd8b6a 100644 --- a/python/paddle/framework/framework.py +++ b/python/paddle/framework/framework.py @@ -19,6 +19,8 @@ import numpy as np from contextlib import contextmanager +__all__ = [] + def set_default_dtype(d): """ diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 955d8610a5909..f84ed941e35fe 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -38,6 +38,8 @@ from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX, INFER_PARAMS_INFO_SUFFIX +__all__ = [] + def _build_saved_state_dict(state_dict): save_dict = {} diff --git a/python/paddle/framework/random.py b/python/paddle/framework/random.py index 251a8407035fd..701f8b5352c3d 100644 --- a/python/paddle/framework/random.py +++ b/python/paddle/framework/random.py @@ -16,6 +16,8 @@ import paddle.fluid as fluid from paddle.fluid import core +__all__ = [] + def seed(seed): """ diff --git a/python/paddle/nn/clip.py b/python/paddle/nn/clip.py index 70c49b4a53876..e868cbdbacc17 100644 --- a/python/paddle/nn/clip.py +++ b/python/paddle/nn/clip.py @@ -16,3 +16,5 @@ from ..fluid.clip import ClipGradByGlobalNorm # noqa: F401 from ..fluid.clip import ClipGradByNorm # noqa: F401 from ..fluid.clip import ClipGradByValue # noqa: F401 + +__all__ = [] diff --git a/python/paddle/nn/decode.py b/python/paddle/nn/decode.py index 3229f0b21a669..ff4a6e4f482af 100644 --- a/python/paddle/nn/decode.py +++ b/python/paddle/nn/decode.py @@ -14,3 +14,5 @@ from ..fluid.layers import BeamSearchDecoder # noqa: F401 from ..fluid.layers import dynamic_decode # noqa: F401 + +__all__ = [] diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index cd8ee99baa237..9001ba16b7ac2 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -27,6 +27,8 @@ from ...fluid.data_feeder import check_variable_and_dtype, check_dtype import paddle +__all__ = [] + def elu(x, alpha=1.0, name=None): r""" diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index 7379c7a5f67bd..65b9c6771c4f1 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -34,6 +34,8 @@ from ...fluid import core, layers from ...fluid.data_feeder import check_variable_and_dtype +__all__ = [] + def interpolate(x, size=None, diff --git a/python/paddle/nn/functional/conv.py b/python/paddle/nn/functional/conv.py index 800c820497372..1edbc5f462ecd 100644 --- a/python/paddle/nn/functional/conv.py +++ b/python/paddle/nn/functional/conv.py @@ -22,6 +22,8 @@ from ...fluid.param_attr import ParamAttr from ...fluid.layer_helper import LayerHelper +__all__ = [] + def _is_list_or_tuple(input): return isinstance(input, (list, tuple)) diff --git a/python/paddle/nn/functional/extension.py b/python/paddle/nn/functional/extension.py index 7900f903e7fd2..8a9597119ab8d 100644 --- a/python/paddle/nn/functional/extension.py +++ b/python/paddle/nn/functional/extension.py @@ -23,6 +23,8 @@ from ...fluid.layers.layer_function_generator import templatedoc from ...fluid.layers.sequence_lod import sequence_mask +__all__ = [] + def diag_embed(input, offset=0, dim1=-2, dim2=-1): """ diff --git a/python/paddle/nn/functional/input.py b/python/paddle/nn/functional/input.py index 4fff9cda4be33..67dc69c1a93b6 100644 --- a/python/paddle/nn/functional/input.py +++ b/python/paddle/nn/functional/input.py @@ -19,6 +19,8 @@ from ...fluid.layers import core from ...fluid.data_feeder import check_variable_and_dtype, check_dtype +__all__ = [] + def one_hot(x, num_classes, name=None): """ diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index bb2d8005f4e31..31ffb91f30dca 100755 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -39,6 +39,8 @@ from ...fluid.framework import Variable from paddle.utils import deprecated +__all__ = [] + def binary_cross_entropy(input, label, weight=None, reduction='mean', name=None): diff --git a/python/paddle/nn/functional/norm.py b/python/paddle/nn/functional/norm.py index dddc4c66d591c..20e3254638997 100644 --- a/python/paddle/nn/functional/norm.py +++ b/python/paddle/nn/functional/norm.py @@ -24,6 +24,8 @@ from ...fluid import core, dygraph_utils import numbers +__all__ = [] + def normalize(x, p=2, axis=1, epsilon=1e-12, name=None): r""" diff --git a/python/paddle/nn/functional/pooling.py b/python/paddle/nn/functional/pooling.py index 27a66c629cafa..1869ac15b17a3 100755 --- a/python/paddle/nn/functional/pooling.py +++ b/python/paddle/nn/functional/pooling.py @@ -18,6 +18,8 @@ from ...fluid.layers import utils, LayerHelper, unsqueeze, squeeze from ...fluid.data_feeder import check_type, check_variable_and_dtype +__all__ = [] + def _is_list_or_tuple(input): return isinstance(input, (list, tuple)) diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index cb8a817023d22..55a66e70160b6 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -19,6 +19,8 @@ from ...fluid import dygraph_utils import numpy as np +__all__ = [] + def affine_grid(theta, out_shape, align_corners=True, name=None): """ diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py index 642919f354075..13a70a179ffe3 100644 --- a/python/paddle/nn/initializer/assign.py +++ b/python/paddle/nn/initializer/assign.py @@ -19,6 +19,8 @@ from ...fluid.data_feeder import check_type from ...fluid.initializer import NumpyArrayInitializer +__all__ = [] + class Assign(NumpyArrayInitializer): """Init an parameter with a numpy array, list, or tensor. diff --git a/python/paddle/nn/initializer/constant.py b/python/paddle/nn/initializer/constant.py index aec3e82aab62b..292eaff362b40 100644 --- a/python/paddle/nn/initializer/constant.py +++ b/python/paddle/nn/initializer/constant.py @@ -15,6 +15,8 @@ # TODO: define the initializers of Constant in neural network from ...fluid.initializer import ConstantInitializer +__all__ = [] + class Constant(ConstantInitializer): """Implement the constant initializer. diff --git a/python/paddle/nn/initializer/kaiming.py b/python/paddle/nn/initializer/kaiming.py index 712bffccda102..f0847c85237b2 100644 --- a/python/paddle/nn/initializer/kaiming.py +++ b/python/paddle/nn/initializer/kaiming.py @@ -15,6 +15,8 @@ # TODO: define the initializers of Kaiming functions in neural network from ...fluid.initializer import MSRAInitializer +__all__ = [] + class KaimingNormal(MSRAInitializer): r"""Implements the Kaiming Normal initializer diff --git a/python/paddle/nn/initializer/normal.py b/python/paddle/nn/initializer/normal.py index c009df780054e..6fee5058057cb 100644 --- a/python/paddle/nn/initializer/normal.py +++ b/python/paddle/nn/initializer/normal.py @@ -15,6 +15,8 @@ from ...fluid.initializer import NormalInitializer from ...fluid.initializer import TruncatedNormalInitializer +__all__ = [] + class Normal(NormalInitializer): """The Random Normal (Gaussian) distribution initializer. diff --git a/python/paddle/nn/initializer/uniform.py b/python/paddle/nn/initializer/uniform.py index e54a4d2187b8d..cac03b5948071 100644 --- a/python/paddle/nn/initializer/uniform.py +++ b/python/paddle/nn/initializer/uniform.py @@ -14,6 +14,8 @@ from ...fluid.initializer import UniformInitializer +__all__ = [] + class Uniform(UniformInitializer): """The random uniform distribution initializer. diff --git a/python/paddle/nn/initializer/xavier.py b/python/paddle/nn/initializer/xavier.py index 01a4a8887b489..f2d5593032f64 100644 --- a/python/paddle/nn/initializer/xavier.py +++ b/python/paddle/nn/initializer/xavier.py @@ -14,6 +14,8 @@ from ...fluid.initializer import XavierInitializer +__all__ = [] + class XavierNormal(XavierInitializer): r""" diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py index 64f0391fb6533..10c2b1e3056f1 100644 --- a/python/paddle/nn/layer/__init__.py +++ b/python/paddle/nn/layer/__init__.py @@ -81,3 +81,5 @@ from .vision import PixelShuffle # noqa: F401 from .distance import PairwiseDistance # noqa: F401 from .container import LayerDict # noqa: F401 + +__all__ = [] diff --git a/python/paddle/nn/layer/activation.py b/python/paddle/nn/layer/activation.py index c6ce4588ea5da..d5b37144cfffe 100644 --- a/python/paddle/nn/layer/activation.py +++ b/python/paddle/nn/layer/activation.py @@ -22,6 +22,8 @@ from paddle.framework import get_default_dtype from .. import functional as F +__all__ = [] + class ELU(layers.Layer): r""" diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py index 058507ba5dec3..f608f20feef55 100644 --- a/python/paddle/nn/layer/common.py +++ b/python/paddle/nn/layer/common.py @@ -20,6 +20,8 @@ from .. import functional as F from ...fluid.framework import _dygraph_tracer +__all__ = [] + def _npairs(x, n): if isinstance(x, (paddle.Tensor, list)): diff --git a/python/paddle/nn/layer/container.py b/python/paddle/nn/layer/container.py index db317839ae818..ad41535f44ad6 100644 --- a/python/paddle/nn/layer/container.py +++ b/python/paddle/nn/layer/container.py @@ -16,7 +16,7 @@ from ...fluid.dygraph.layers import Layer from six.moves import collections_abc -__all__ = ['LayerDict', ] +__all__ = [] class LayerDict(Layer): diff --git a/python/paddle/nn/layer/conv.py b/python/paddle/nn/layer/conv.py index 2360dc17cf171..2de065d62a4f8 100644 --- a/python/paddle/nn/layer/conv.py +++ b/python/paddle/nn/layer/conv.py @@ -25,6 +25,8 @@ from ...fluid.layers import utils from ..functional.conv import _update_padding_nd +__all__ = [] + def _get_default_param_initializer(num_channels, filter_size): filter_elem_num = num_channels * np.prod(filter_size) diff --git a/python/paddle/nn/layer/distance.py b/python/paddle/nn/layer/distance.py index 7eb0fc1fbb575..77e3447ffda00 100644 --- a/python/paddle/nn/layer/distance.py +++ b/python/paddle/nn/layer/distance.py @@ -20,6 +20,8 @@ from ...fluid.data_feeder import check_variable_and_dtype, check_type from ...fluid.layer_helper import LayerHelper +__all__ = [] + class PairwiseDistance(layers.Layer): r""" diff --git a/python/paddle/nn/layer/loss.py b/python/paddle/nn/layer/loss.py index 356b22c632cf5..8f43eb8866b4b 100644 --- a/python/paddle/nn/layer/loss.py +++ b/python/paddle/nn/layer/loss.py @@ -21,6 +21,8 @@ from .. import functional as F from paddle.fluid.framework import core, in_dygraph_mode, _varbase_creator +__all__ = [] + class BCEWithLogitsLoss(fluid.dygraph.Layer): r""" diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py index 970d68e826343..45640a6598e57 100644 --- a/python/paddle/nn/layer/norm.py +++ b/python/paddle/nn/layer/norm.py @@ -50,6 +50,8 @@ from ...fluid.dygraph.base import no_grad from .. import functional as F +__all__ = [] + class _InstanceNormBase(layers.Layer): """ diff --git a/python/paddle/nn/layer/pooling.py b/python/paddle/nn/layer/pooling.py index 5916fd7c69eb0..528572ee21b7c 100755 --- a/python/paddle/nn/layer/pooling.py +++ b/python/paddle/nn/layer/pooling.py @@ -16,6 +16,8 @@ from ...fluid.layer_helper import LayerHelper from .. import functional as F +__all__ = [] + class AvgPool1D(layers.Layer): r""" diff --git a/python/paddle/nn/layer/rnn.py b/python/paddle/nn/layer/rnn.py index a7539b5b09571..de9b8cdbfce2a 100644 --- a/python/paddle/nn/layer/rnn.py +++ b/python/paddle/nn/layer/rnn.py @@ -33,6 +33,8 @@ from paddle.fluid.layers.utils import map_structure, flatten, pack_sequence_as from paddle.fluid.data_feeder import convert_dtype +__all__ = [] + def split_states(states, bidirectional=False, state_components=1): r""" diff --git a/python/paddle/nn/layer/transformer.py b/python/paddle/nn/layer/transformer.py index 752870f3d0a28..891177532a438 100644 --- a/python/paddle/nn/layer/transformer.py +++ b/python/paddle/nn/layer/transformer.py @@ -28,6 +28,8 @@ from ...fluid.param_attr import ParamAttr from ...fluid.data_feeder import convert_dtype +__all__ = [] + def _convert_param_attr_to_list(param_attr, n): """ diff --git a/python/paddle/nn/layer/vision.py b/python/paddle/nn/layer/vision.py index e66e122be5259..e6d3af9a37b32 100644 --- a/python/paddle/nn/layer/vision.py +++ b/python/paddle/nn/layer/vision.py @@ -17,6 +17,8 @@ from ...fluid.dygraph import layers from .. import functional +__all__ = [] + class PixelShuffle(layers.Layer): """ diff --git a/python/paddle/nn/utils/weight_norm_hook.py b/python/paddle/nn/utils/weight_norm_hook.py index 23df38ca08c45..8d2cc8062d2cc 100755 --- a/python/paddle/nn/utils/weight_norm_hook.py +++ b/python/paddle/nn/utils/weight_norm_hook.py @@ -19,6 +19,8 @@ from ...fluid.layer_helper import LayerHelper from ...fluid.data_feeder import check_variable_and_dtype +__all__ = [] + def l2_norm(x, axis, epsilon=1e-12, name=None): if len(x.shape) == 1: diff --git a/python/paddle/optimizer/adadelta.py b/python/paddle/optimizer/adadelta.py index af07d706e135d..6c10d9bc2690a 100644 --- a/python/paddle/optimizer/adadelta.py +++ b/python/paddle/optimizer/adadelta.py @@ -17,6 +17,8 @@ from ..fluid import framework from ..fluid.framework import Variable, name_scope +__all__ = [] + class Adadelta(Optimizer): r""" diff --git a/python/paddle/optimizer/adagrad.py b/python/paddle/optimizer/adagrad.py index 82615c92b7cfe..bb934e5a9262c 100644 --- a/python/paddle/optimizer/adagrad.py +++ b/python/paddle/optimizer/adagrad.py @@ -17,6 +17,8 @@ from ..fluid import framework from ..fluid.framework import Variable +__all__ = [] + class Adagrad(Optimizer): r""" diff --git a/python/paddle/optimizer/adam.py b/python/paddle/optimizer/adam.py index 4904ebb56cc91..75803e8cc07bc 100644 --- a/python/paddle/optimizer/adam.py +++ b/python/paddle/optimizer/adam.py @@ -24,6 +24,8 @@ import paddle +__all__ = [] + class Adam(Optimizer): r""" diff --git a/python/paddle/optimizer/adamax.py b/python/paddle/optimizer/adamax.py index 175d932540dee..44ae89f49d1c0 100644 --- a/python/paddle/optimizer/adamax.py +++ b/python/paddle/optimizer/adamax.py @@ -17,6 +17,8 @@ from ..fluid import framework from ..fluid.framework import Variable, name_scope +__all__ = [] + class Adamax(Optimizer): r""" diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py index 899c2957a6a4f..304f0b771826c 100644 --- a/python/paddle/optimizer/adamw.py +++ b/python/paddle/optimizer/adamw.py @@ -19,6 +19,8 @@ from ..fluid.dygraph import base as imperative_base import paddle +__all__ = [] + class AdamW(Adam): r""" diff --git a/python/paddle/optimizer/lamb.py b/python/paddle/optimizer/lamb.py index bab130ec59098..bff24e71c8153 100644 --- a/python/paddle/optimizer/lamb.py +++ b/python/paddle/optimizer/lamb.py @@ -17,6 +17,8 @@ from ..fluid import framework from ..fluid.framework import Variable +__all__ = [] + class Lamb(Optimizer): r""" diff --git a/python/paddle/optimizer/momentum.py b/python/paddle/optimizer/momentum.py index c1dc0e8ddd8af..372143553e0c3 100644 --- a/python/paddle/optimizer/momentum.py +++ b/python/paddle/optimizer/momentum.py @@ -22,6 +22,8 @@ import paddle.fluid as fluid from paddle.fluid.regularizer import L2DecayRegularizer +__all__ = [] + class Momentum(Optimizer): r""" diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index 9425ab1431e70..b06bd2a2b0be9 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -42,6 +42,8 @@ from .. import compat as cpt from .lr import LRScheduler +__all__ = [] + class Optimizer(object): r"""Optimizer Base class. diff --git a/python/paddle/optimizer/rmsprop.py b/python/paddle/optimizer/rmsprop.py index a2fd40bc0b369..b0bb0228c8ca8 100644 --- a/python/paddle/optimizer/rmsprop.py +++ b/python/paddle/optimizer/rmsprop.py @@ -17,6 +17,8 @@ from ..fluid import framework from ..fluid.framework import Variable +__all__ = [] + class RMSProp(Optimizer): r""" diff --git a/python/paddle/optimizer/sgd.py b/python/paddle/optimizer/sgd.py index ecac40aec7298..4526034b405b0 100644 --- a/python/paddle/optimizer/sgd.py +++ b/python/paddle/optimizer/sgd.py @@ -18,6 +18,8 @@ from ..fluid.framework import Variable, name_scope from ..fluid.dygraph import no_grad +__all__ = [] + class SGD(Optimizer): r""" diff --git a/python/paddle/proto/__init__.py b/python/paddle/proto/__init__.py index 07406a841ec90..f482d80548de1 100644 --- a/python/paddle/proto/__init__.py +++ b/python/paddle/proto/__init__.py @@ -14,3 +14,5 @@ from paddle.proto.TrainerConfig_pb2 import OptimizationConfig, TrainerConfig from paddle.proto.ModelConfig_pb2 import ModelConfig + +__all__ = [] diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index 0aefcf9e683da..3129029d82920 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -27,6 +27,8 @@ import zlib import paddle.compat as cpt +__all__ = [] + # On macOS, the 'spawn' start method is now the default in Python3.8 multiprocessing, # Paddle is currently unable to solve this, so forces the process to start using # the 'fork' start method. diff --git a/python/paddle/reader/tests/decorator_test.py b/python/paddle/reader/tests/decorator_test.py index e15702e39c458..e11600a06fb9e 100644 --- a/python/paddle/reader/tests/decorator_test.py +++ b/python/paddle/reader/tests/decorator_test.py @@ -19,6 +19,8 @@ import paddle.reader +__all__ = [] + def reader_creator_10(dur): def reader(): diff --git a/python/paddle/static/amp/__init__.py b/python/paddle/static/amp/__init__.py index 7320efe9b1799..54de11401f3c6 100644 --- a/python/paddle/static/amp/__init__.py +++ b/python/paddle/static/amp/__init__.py @@ -22,3 +22,5 @@ from ...fluid.contrib.mixed_precision import bf16_guard # noqa: F401 from ...fluid.contrib.mixed_precision import rewrite_program_bf16 # noqa: F401 from ...fluid.contrib.mixed_precision import convert_float_to_uint16 # noqa: F401 + +__all__ = [] diff --git a/python/paddle/static/input.py b/python/paddle/static/input.py index c1de576ee74c9..f06c45cc36973 100644 --- a/python/paddle/static/input.py +++ b/python/paddle/static/input.py @@ -21,6 +21,8 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ from paddle.fluid.framework import static_only +__all__ = [] + @static_only def data(name, shape, dtype=None, lod_level=0): diff --git a/python/paddle/static/io.py b/python/paddle/static/io.py index fc6d8b64f18cb..58e8ebc481d79 100644 --- a/python/paddle/static/io.py +++ b/python/paddle/static/io.py @@ -37,6 +37,8 @@ from paddle.fluid.executor import Executor, global_scope from paddle.fluid.log_helper import get_logger +__all__ = [] + _logger = get_logger( __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s') diff --git a/python/paddle/static/nn/common.py b/python/paddle/static/nn/common.py index 659b7f45b26a7..b8133872aa934 100755 --- a/python/paddle/static/nn/common.py +++ b/python/paddle/static/nn/common.py @@ -15,6 +15,8 @@ import paddle from paddle.fluid.framework import static_only +__all__ = [] + @static_only def fc(x, diff --git a/python/paddle/tensor/array.py b/python/paddle/tensor/array.py index ee28d47a9a9fd..6c3d5c577e745 100644 --- a/python/paddle/tensor/array.py +++ b/python/paddle/tensor/array.py @@ -16,6 +16,8 @@ from ..fluid import layers +__all__ = [] + def array_length(array): """ diff --git a/python/paddle/tensor/attribute.py b/python/paddle/tensor/attribute.py index 1f709ac4dbc86..131afca0d676d 100644 --- a/python/paddle/tensor/attribute.py +++ b/python/paddle/tensor/attribute.py @@ -22,6 +22,8 @@ from ..fluid.layers import rank # noqa: F401 from ..fluid.layers import shape # noqa: F401 +__all__ = [] + def _complex_to_real_dtype(dtype): if dtype == core.VarDesc.VarType.COMPLEX64: diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py index b31984f684695..361c0e80f90d7 100644 --- a/python/paddle/tensor/creation.py +++ b/python/paddle/tensor/creation.py @@ -31,6 +31,8 @@ from ..fluid.layers import linspace # noqa: F401 import paddle +__all__ = [] + @dygraph_only def to_tensor(data, dtype=None, place=None, stop_gradient=True): diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index 87e3bce4b1d69..8aa9c9bd2bd7f 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -21,6 +21,8 @@ from paddle.common_ops_import import core from paddle.common_ops_import import VarDesc +__all__ = [] + def matmul(x, y, transpose_x=False, transpose_y=False, name=None): """ diff --git a/python/paddle/tensor/logic.py b/python/paddle/tensor/logic.py index 14154fb06f83e..bdf2c477d8658 100644 --- a/python/paddle/tensor/logic.py +++ b/python/paddle/tensor/logic.py @@ -28,6 +28,8 @@ from paddle.common_ops_import import core +__all__ = [] + def equal_all(x, y, name=None): """ diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index dc811ea0f3fa6..1a5962042675d 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -34,6 +34,8 @@ import paddle import warnings +__all__ = [] + def _print_warning_in_static_mode(api_name): warnings.warn( diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index 65f57b4b4e93b..84c67a9ae8d9d 100755 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -59,6 +59,8 @@ from ..fluid.layers import multiplex # noqa: F401 from ..fluid import layers +__all__ = [] + _supported_int_dtype_ = [ VarDesc.VarType.UINT8, VarDesc.VarType.INT8, diff --git a/python/paddle/tensor/random.py b/python/paddle/tensor/random.py index 7e1eef8f32508..69a4634544763 100644 --- a/python/paddle/tensor/random.py +++ b/python/paddle/tensor/random.py @@ -21,6 +21,8 @@ from ..fluid.layers import utils import paddle +__all__ = [] + def bernoulli(x, name=None): """ diff --git a/python/paddle/tensor/search.py b/python/paddle/tensor/search.py index ac303d2311eb9..3d8a75f9277af 100644 --- a/python/paddle/tensor/search.py +++ b/python/paddle/tensor/search.py @@ -25,6 +25,8 @@ # from ..fluid.layers import has_inf #DEFINE_ALIAS # from ..fluid.layers import has_nan #DEFINE_ALIAS +__all__ = [] + def argsort(x, axis=-1, descending=False, name=None): """ diff --git a/python/paddle/tensor/stat.py b/python/paddle/tensor/stat.py index fa7a278a2b52c..8c74360a17d05 100644 --- a/python/paddle/tensor/stat.py +++ b/python/paddle/tensor/stat.py @@ -23,6 +23,8 @@ from ..fluid.data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype import paddle +__all__ = [] + def mean(x, axis=None, keepdim=False, name=None): """ diff --git a/python/paddle/tensor/to_string.py b/python/paddle/tensor/to_string.py index 2e76a8d47a773..9d07840be6882 100644 --- a/python/paddle/tensor/to_string.py +++ b/python/paddle/tensor/to_string.py @@ -17,6 +17,8 @@ from paddle.fluid.layers import core from paddle.fluid.data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +__all__ = [] + class PrintOptions(object): precision = 8 diff --git a/python/paddle/tests/test_dataset_cifar.py b/python/paddle/tests/test_dataset_cifar.py index e84f73188666a..abf79fb1e3974 100644 --- a/python/paddle/tests/test_dataset_cifar.py +++ b/python/paddle/tests/test_dataset_cifar.py @@ -15,7 +15,7 @@ import unittest import numpy as np -from paddle.vision.datasets import * +from paddle.vision.datasets import Cifar10, Cifar100 class TestCifar10Train(unittest.TestCase): diff --git a/python/paddle/tests/test_dataset_conll05.py b/python/paddle/tests/test_dataset_conll05.py index e35c04275d204..9eb0036718b35 100644 --- a/python/paddle/tests/test_dataset_conll05.py +++ b/python/paddle/tests/test_dataset_conll05.py @@ -16,7 +16,7 @@ import unittest import numpy as np -from paddle.text.datasets import * +from paddle.text.datasets import Conll05st class TestConll05st(unittest.TestCase): diff --git a/python/paddle/tests/test_dataset_imdb.py b/python/paddle/tests/test_dataset_imdb.py index 62c75ab232c8d..aed8c387409dc 100644 --- a/python/paddle/tests/test_dataset_imdb.py +++ b/python/paddle/tests/test_dataset_imdb.py @@ -15,7 +15,7 @@ import unittest import numpy as np -from paddle.text.datasets import * +from paddle.text.datasets import Imdb class TestImdbTrain(unittest.TestCase): diff --git a/python/paddle/tests/test_dataset_imikolov.py b/python/paddle/tests/test_dataset_imikolov.py index f4f0b8e483677..6ffeeda73c362 100644 --- a/python/paddle/tests/test_dataset_imikolov.py +++ b/python/paddle/tests/test_dataset_imikolov.py @@ -15,7 +15,7 @@ import unittest import numpy as np -from paddle.text.datasets import * +from paddle.text.datasets import Imikolov class TestImikolovTrain(unittest.TestCase): diff --git a/python/paddle/tests/test_dataset_movielens.py b/python/paddle/tests/test_dataset_movielens.py index 3b61fd6f5c7c2..e5c6d8376eed9 100644 --- a/python/paddle/tests/test_dataset_movielens.py +++ b/python/paddle/tests/test_dataset_movielens.py @@ -15,7 +15,7 @@ import unittest import numpy as np -from paddle.text.datasets import * +from paddle.text.datasets import Movielens class TestMovielensTrain(unittest.TestCase): diff --git a/python/paddle/tests/test_dataset_uci_housing.py b/python/paddle/tests/test_dataset_uci_housing.py index 623c7d24d09da..bdf960b433687 100644 --- a/python/paddle/tests/test_dataset_uci_housing.py +++ b/python/paddle/tests/test_dataset_uci_housing.py @@ -19,7 +19,7 @@ import shutil import cv2 -from paddle.text.datasets import * +from paddle.text.datasets import UCIHousing, WMT14 class TestUCIHousingTrain(unittest.TestCase): diff --git a/python/paddle/tests/test_dataset_wmt.py b/python/paddle/tests/test_dataset_wmt.py index b4945cb90f991..3e63090c9f0ff 100644 --- a/python/paddle/tests/test_dataset_wmt.py +++ b/python/paddle/tests/test_dataset_wmt.py @@ -15,7 +15,7 @@ import unittest import numpy as np -from paddle.text.datasets import * +from paddle.text.datasets import WMT14, WMT16 class TestWMT14Train(unittest.TestCase): diff --git a/python/paddle/tests/test_datasets.py b/python/paddle/tests/test_datasets.py index 89fa01cbceb45..c93bac3ac27e8 100644 --- a/python/paddle/tests/test_datasets.py +++ b/python/paddle/tests/test_datasets.py @@ -20,7 +20,7 @@ import cv2 import paddle.vision.transforms as T -from paddle.vision.datasets import * +from paddle.vision.datasets import DatasetFolder, ImageFolder, MNIST, FashionMNIST, Flowers from paddle.dataset.common import _check_exists_and_download diff --git a/python/paddle/text/datasets/__init__.py b/python/paddle/text/datasets/__init__.py index 9a00081469a8b..118917049928b 100644 --- a/python/paddle/text/datasets/__init__.py +++ b/python/paddle/text/datasets/__init__.py @@ -19,3 +19,5 @@ from .uci_housing import UCIHousing # noqa: F401 from .wmt14 import WMT14 # noqa: F401 from .wmt16 import WMT16 # noqa: F401 + +__all__ = [] diff --git a/python/paddle/text/datasets/conll05.py b/python/paddle/text/datasets/conll05.py index 070c787db8574..7dd29637706f3 100644 --- a/python/paddle/text/datasets/conll05.py +++ b/python/paddle/text/datasets/conll05.py @@ -24,6 +24,8 @@ import paddle.compat as cpt from paddle.dataset.common import _check_exists_and_download +__all__ = [] + DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz' DATA_MD5 = '387719152ae52d60422c016e92a742fc' WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt' diff --git a/python/paddle/text/datasets/imdb.py b/python/paddle/text/datasets/imdb.py index c64890dc43d77..f4fe7eb174bb7 100644 --- a/python/paddle/text/datasets/imdb.py +++ b/python/paddle/text/datasets/imdb.py @@ -24,6 +24,8 @@ from paddle.io import Dataset from paddle.dataset.common import _check_exists_and_download +__all__ = [] + URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz' MD5 = '7c2ac02c03563afcf9b574c7e56c153a' diff --git a/python/paddle/text/datasets/imikolov.py b/python/paddle/text/datasets/imikolov.py index 7e4daf731a23a..9c84669d6b8d8 100644 --- a/python/paddle/text/datasets/imikolov.py +++ b/python/paddle/text/datasets/imikolov.py @@ -22,6 +22,8 @@ from paddle.io import Dataset from paddle.dataset.common import _check_exists_and_download +__all__ = [] + URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz' MD5 = '30177ea32e27c525793142b6bf2c8e2d' diff --git a/python/paddle/text/datasets/movielens.py b/python/paddle/text/datasets/movielens.py index 7741e82194ca7..798a7c590e17b 100644 --- a/python/paddle/text/datasets/movielens.py +++ b/python/paddle/text/datasets/movielens.py @@ -26,6 +26,8 @@ import paddle.compat as cpt from paddle.dataset.common import _check_exists_and_download +__all__ = [] + age_table = [1, 18, 25, 35, 45, 50, 56] URL = 'https://dataset.bj.bcebos.com/movielens%2Fml-1m.zip' diff --git a/python/paddle/text/datasets/uci_housing.py b/python/paddle/text/datasets/uci_housing.py index c876ed409cf99..597b1e1e8185e 100644 --- a/python/paddle/text/datasets/uci_housing.py +++ b/python/paddle/text/datasets/uci_housing.py @@ -21,6 +21,8 @@ from paddle.io import Dataset from paddle.dataset.common import _check_exists_and_download +__all__ = [] + URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data' MD5 = 'd4accdce7a25600298819f8e28e8d593' feature_names = [ diff --git a/python/paddle/text/datasets/wmt14.py b/python/paddle/text/datasets/wmt14.py index 96d29c79c6a9d..424a564216d19 100644 --- a/python/paddle/text/datasets/wmt14.py +++ b/python/paddle/text/datasets/wmt14.py @@ -22,6 +22,8 @@ import paddle.compat as cpt from paddle.dataset.common import _check_exists_and_download +__all__ = [] + URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' 'cslm_joint_paper/data/dev+test.tgz') MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' diff --git a/python/paddle/text/datasets/wmt16.py b/python/paddle/text/datasets/wmt16.py index 5605fd2aecbdc..f95cbe771cadc 100644 --- a/python/paddle/text/datasets/wmt16.py +++ b/python/paddle/text/datasets/wmt16.py @@ -27,6 +27,8 @@ import paddle.compat as cpt from paddle.dataset.common import _check_exists_and_download +__all__ = [] + DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz") DATA_MD5 = "0c38be43600334966403524a40dcd81e" diff --git a/python/paddle/utils/deprecated.py b/python/paddle/utils/deprecated.py index a46f1ae3a2c2e..5390dea69fe7d 100755 --- a/python/paddle/utils/deprecated.py +++ b/python/paddle/utils/deprecated.py @@ -19,6 +19,8 @@ import functools import paddle +__all__ = [] + # NOTE(zhiqiu): Since python 3.2, DeprecationWarning is ignored by default, # and since python 3.7, it is once again shown by default when triggered directly by code in __main__. # See details: https://docs.python.org/3/library/warnings.html#default-warning-filter diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index bd70013e1120e..ddd1dad9dbdf5 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -55,6 +55,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): import logging logger = logging.getLogger(__name__) +__all__ = [] + WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/hapi/weights") DOWNLOAD_RETRY_LIMIT = 3 diff --git a/python/paddle/utils/image_util.py b/python/paddle/utils/image_util.py index b113f574e9fac..18be9366c40a7 100644 --- a/python/paddle/utils/image_util.py +++ b/python/paddle/utils/image_util.py @@ -16,6 +16,8 @@ from PIL import Image from six.moves import cStringIO as StringIO +__all__ = [] + def resize_image(img, target_size): """ diff --git a/python/paddle/utils/install_check.py b/python/paddle/utils/install_check.py index 5d70cf61007a6..69baa4facfa96 100644 --- a/python/paddle/utils/install_check.py +++ b/python/paddle/utils/install_check.py @@ -20,6 +20,8 @@ import paddle +__all__ = [] + def _simple_network(): """ diff --git a/python/paddle/utils/lazy_import.py b/python/paddle/utils/lazy_import.py index ea07077b2da2a..d9146422819f8 100644 --- a/python/paddle/utils/lazy_import.py +++ b/python/paddle/utils/lazy_import.py @@ -15,6 +15,8 @@ import importlib +__all__ = [] + def try_import(module_name): """Try importing a module, with an informative error message on failure.""" diff --git a/python/paddle/utils/op_version.py b/python/paddle/utils/op_version.py index a1fa230d64faa..6e81b5a2c17bb 100644 --- a/python/paddle/utils/op_version.py +++ b/python/paddle/utils/op_version.py @@ -14,6 +14,8 @@ from ..fluid import core +__all__ = [] + def Singleton(cls): _instance = {} From 0e904d489c7123505b69ec85cde1a8dc8c196591 Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Thu, 29 Apr 2021 10:29:23 +0800 Subject: [PATCH 007/156] update 2.0 public api in hapi (#32651) --- python/paddle/hapi/__init__.py | 19 +++++++++---------- python/paddle/hapi/dynamic_flops.py | 2 +- python/paddle/hapi/hub.py | 2 ++ python/paddle/hapi/logger.py | 2 ++ python/paddle/hapi/model.py | 2 +- python/paddle/hapi/model_summary.py | 2 +- python/paddle/hapi/progressbar.py | 2 +- python/paddle/hapi/static_flops.py | 2 ++ 8 files changed, 19 insertions(+), 14 deletions(-) diff --git a/python/paddle/hapi/__init__.py b/python/paddle/hapi/__init__.py index 6b7672828e63d..2829bbe947089 100644 --- a/python/paddle/hapi/__init__.py +++ b/python/paddle/hapi/__init__.py @@ -12,17 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import logger -from . import callbacks -from . import model_summary -from . import hub +from . import logger # noqa: F401 +from . import callbacks # noqa: F401 +from . import hub # noqa: F401 +from . import progressbar # noqa: F401 +from . import static_flops # noqa: F401 -from . import model -from .model import * -from .model_summary import summary -from .dynamic_flops import flops +from .model import Model # noqa: F401 +from .model_summary import summary # noqa: F401 +from .dynamic_flops import flops # noqa: F401 logger.setup_logger() -__all__ = ['callbacks'] + model.__all__ + ['summary'] -__all__ = model.__all__ + ['flops'] +__all__ = [] diff --git a/python/paddle/hapi/dynamic_flops.py b/python/paddle/hapi/dynamic_flops.py index 35819d6b7bb55..8be6758f1e54b 100644 --- a/python/paddle/hapi/dynamic_flops.py +++ b/python/paddle/hapi/dynamic_flops.py @@ -18,7 +18,7 @@ import numpy as np from .static_flops import static_flops, Table -__all__ = ['flops'] +__all__ = [] def flops(net, input_size, custom_ops=None, print_detail=False): diff --git a/python/paddle/hapi/hub.py b/python/paddle/hapi/hub.py index 31a8be0944f3d..6490c878f9b88 100644 --- a/python/paddle/hapi/hub.py +++ b/python/paddle/hapi/hub.py @@ -19,6 +19,8 @@ import zipfile from paddle.utils.download import get_path_from_url +__all__ = [] + DEFAULT_CACHE_DIR = '~/.cache' VAR_DEPENDENCY = 'dependencies' MODULE_HUBCONF = 'hubconf.py' diff --git a/python/paddle/hapi/logger.py b/python/paddle/hapi/logger.py index d4f18ce0ff738..ea515d9532467 100644 --- a/python/paddle/hapi/logger.py +++ b/python/paddle/hapi/logger.py @@ -22,6 +22,8 @@ from paddle.fluid.dygraph.parallel import ParallelEnv +__all__ = [] + def setup_logger(output=None, name="hapi", log_level=logging.INFO): """ diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 5a33d5b58dc1a..160d6c54759d9 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -54,7 +54,7 @@ from .callbacks import config_callbacks, EarlyStopping from .model_summary import summary -__all__ = ['Model', ] +__all__ = [] _parallel_context_initialized = False diff --git a/python/paddle/hapi/model_summary.py b/python/paddle/hapi/model_summary.py index 9f2769e1ca285..d78196d94451e 100644 --- a/python/paddle/hapi/model_summary.py +++ b/python/paddle/hapi/model_summary.py @@ -22,7 +22,7 @@ from collections import OrderedDict -__all__ = ['summary'] +__all__ = [] def summary(net, input_size, dtypes=None): diff --git a/python/paddle/hapi/progressbar.py b/python/paddle/hapi/progressbar.py index cf5a03ed4982b..5f63a3169f8ac 100644 --- a/python/paddle/hapi/progressbar.py +++ b/python/paddle/hapi/progressbar.py @@ -22,7 +22,7 @@ import numpy as np from collections import namedtuple -__all__ = ['ProgressBar'] +__all__ = [] class ProgressBar(object): diff --git a/python/paddle/hapi/static_flops.py b/python/paddle/hapi/static_flops.py index 3656e0c18945a..07fc19b2cb89a 100644 --- a/python/paddle/hapi/static_flops.py +++ b/python/paddle/hapi/static_flops.py @@ -18,6 +18,8 @@ from collections import OrderedDict from paddle.static import Program, program_guard, Variable +__all__ = [] + class VarWrapper(object): def __init__(self, var, graph): From 7ae0a80f6d6f57f6daf42be6cdb2de59402779a7 Mon Sep 17 00:00:00 2001 From: Jacek Czaja Date: Thu, 29 Apr 2021 05:18:41 +0200 Subject: [PATCH 008/156] - Added clearing oneDNN per executor (#32664) - Executor is nt always having FLAGS_use_mkldnn set to true --- paddle/fluid/framework/executor.cc | 9 ++++-- paddle/fluid/framework/naive_executor.cc | 2 +- .../fluid/inference/api/mkldnn_quantizer.cc | 3 +- .../operators/mkldnn/test_mkldnn_caching.cc | 2 +- paddle/fluid/platform/device_context.cc | 30 ++++++++++++++++--- paddle/fluid/platform/device_context.h | 14 ++++++++- paddle/fluid/platform/mkldnn_helper.h | 8 +++-- 7 files changed, 56 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc index e5bfbf4a8f779..de007c128d754 100644 --- a/paddle/fluid/framework/executor.cc +++ b/paddle/fluid/framework/executor.cc @@ -72,7 +72,7 @@ Executor::~Executor() { #ifdef PADDLE_WITH_MKLDNN // Clear mkl-dnn cache, // this is needed to have mkl-dnn unit tests working - ClearMKLDNNCache(place_); + ClearMKLDNNCache(place_, this); #endif } @@ -169,6 +169,9 @@ void Executor::Run(const ProgramDesc& pdesc, Scope* scope, int block_id, bool force_disable_gc, bool keep_kid_scopes) { platform::RecordBlock b(block_id); if (FLAGS_use_mkldnn) EnableMKLDNN(pdesc); +#ifdef PADDLE_WITH_MKLDNN + platform::AttachPointerHashToMKLDNNKey(this, place_); +#endif auto ctx = Prepare(pdesc, block_id, skip_ref_cnt_vars, force_disable_gc); RunPreparedContext(ctx.get(), scope, create_local_scope, create_vars, keep_kid_scopes); @@ -294,6 +297,9 @@ void Executor::Run(const ProgramDesc& program, Scope* scope, const std::string& fetch_holder_name) { platform::RecordBlock b(kProgramId); if (FLAGS_use_mkldnn) EnableMKLDNN(program); +#ifdef PADDLE_WITH_MKLDNN + platform::AttachPointerHashToMKLDNNKey(this, place_); +#endif bool has_feed_ops = has_feed_operators(program.Block(0), *feed_targets, feed_holder_name); bool has_fetch_ops = @@ -576,7 +582,6 @@ void Executor::EnableMKLDNN(const ProgramDesc& program) { } } } - platform::AttachPointerHashToMKLDNNKey(this, place_); #else LOG(WARNING) << "'MKLDNN' is not supported, Please re-compile with WITH_MKLDNN option"; diff --git a/paddle/fluid/framework/naive_executor.cc b/paddle/fluid/framework/naive_executor.cc index f107321958ba7..7d55d8c41e3e9 100644 --- a/paddle/fluid/framework/naive_executor.cc +++ b/paddle/fluid/framework/naive_executor.cc @@ -128,7 +128,7 @@ NaiveExecutor::~NaiveExecutor() { #ifdef PADDLE_WITH_MKLDNN // Clear mkl-dnn cache, // this is needed to have mkl-dnn unit tests working - ClearMKLDNNCache(place_); + ClearMKLDNNCache(place_, this); #endif } diff --git a/paddle/fluid/inference/api/mkldnn_quantizer.cc b/paddle/fluid/inference/api/mkldnn_quantizer.cc index 793fc53d90b76..f6cdbb00b5045 100644 --- a/paddle/fluid/inference/api/mkldnn_quantizer.cc +++ b/paddle/fluid/inference/api/mkldnn_quantizer.cc @@ -411,7 +411,8 @@ void AnalysisPredictor::MkldnnQuantizer::ClearDeviceContext() const { platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); platform::MKLDNNDeviceContext* dev_ctx = (platform::MKLDNNDeviceContext*)pool.Get(predictor_.place_); - dev_ctx->ResetBlobMap(); + dev_ctx->ResetBlobMap( + paddle::platform::MKLDNNDeviceContext::tls().get_curr_exec()); } void AnalysisPredictor::MkldnnQuantizer::PrepareArgument() const { diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc b/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc index aafff5248a024..d6cd76b697f51 100644 --- a/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc +++ b/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc @@ -50,7 +50,7 @@ class CacheTester { platform::CPUPlace place; onednn_dev_ctx_ = dynamic_cast(pool.Get(place)); - onednn_dev_ctx_->ResetBlobMap(); + onednn_dev_ctx_->ResetBlobMap(nullptr); } bool Analyze(unsigned short int num_entries) { diff --git a/paddle/fluid/platform/device_context.cc b/paddle/fluid/platform/device_context.cc index 50bb64d557444..9a47ac45462ed 100644 --- a/paddle/fluid/platform/device_context.cc +++ b/paddle/fluid/platform/device_context.cc @@ -537,6 +537,7 @@ Place CUDAPinnedDeviceContext::GetPlace() const { return place_; } MKLDNNDeviceContext::MKLDNNDeviceContext(CPUPlace place) : CPUDeviceContext(place), p_blobmap_() { p_blobmap_.reset(new BlobMap()); + p_exec_items_.reset(new ExecMap()); p_mutex_.reset(new std::mutex()); } @@ -560,7 +561,7 @@ MKLDNNDeviceContextThreadLocals::Body::~Body() { platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); platform::MKLDNNDeviceContext* dev_ctx = (platform::MKLDNNDeviceContext*)pool.Get(cpu_place); - dev_ctx->ResetBlobMap(); + dev_ctx->ResetBlobMap(exec_ptr_); } void MKLDNNDeviceContextThreadLocals::Body::set_cur_mkldnn_session_id( @@ -607,17 +608,34 @@ mkldnn::stream& MKLDNNDeviceContextThreadLocals::Body::get_stream(void) { return cur_stream; } -void MKLDNNDeviceContext::ResetBlobMap() { +void MKLDNNDeviceContext::ResetBlobMap(void* ptr) { std::lock_guard lock(*p_mutex_); if (!block_next_cache_clearing_) { VLOG(3) << "Clearing DNNL cache."; - p_blobmap_->clear(); + // If no specific executor pointer then clear + // everything. For executor pointer then clear only + // objects allocated when using given executor + if (ptr == nullptr) { + p_blobmap_->clear(); + } else { + for (auto& v : (*p_exec_items_)[ptr]) { + (v.first)->erase(v.second); + } + p_exec_items_->erase(ptr); + } } else { VLOG(3) << "Prevented Clearing DNNL cache."; block_next_cache_clearing_ = false; } } +void MKLDNNDeviceContext::LinkEntryWithExecutor(BlobPtr_t pblob, + KeyBlob::iterator it) const { + // Take current executor addess from TLS + // and for this executor's items add the one defined with arguments + (*p_exec_items_)[tls().get_curr_exec()].push_back(std::make_pair(pblob, it)); +} + void MKLDNNDeviceContext::BlockNextCacheClearing() { std::lock_guard lock(*p_mutex_); VLOG(3) << "Next DNNL cache clearing has been blocked."; @@ -682,7 +700,11 @@ void MKLDNNDeviceContext::SetBlob(const std::string& name, // Find Blob via name auto blob_it = pBlob->find(name); if (blob_it == pBlob->end()) { - (*pBlob)[name] = data; + auto el = + pBlob->insert(std::make_pair(name, data)); // (*pBlob)[name] = data; + // Register new element in per executor map + // to have easily erased when executor terminated + LinkEntryWithExecutor(pBlob, el.first); } else { blob_it->second = data; // set data to existing blob } diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h index f79cb1ab94788..d91e14ec3aa92 100644 --- a/paddle/fluid/platform/device_context.h +++ b/paddle/fluid/platform/device_context.h @@ -673,6 +673,7 @@ class MKLDNNDeviceContextThreadLocals { mkldnn::stream cur_stream; std::string key_suffix; // Key identifying current Executor bool key_attach_thread_id = true; + void* exec_ptr_ = nullptr; Body(); ~Body(); @@ -689,6 +690,8 @@ class MKLDNNDeviceContextThreadLocals { const std::string& get_key_suffix(void) const { return key_suffix; } void disable_tid_in_key(void) { key_attach_thread_id = false; } bool is_tid_used_in_key(void) const { return key_attach_thread_id; } + void set_curr_exec(void* exec_ptr) { exec_ptr_ = exec_ptr; } + void* get_curr_exec(void) const { return exec_ptr_; } }; MKLDNNDeviceContextThreadLocals() = default; MKLDNNDeviceContextThreadLocals(const MKLDNNDeviceContextThreadLocals& c) = @@ -724,13 +727,19 @@ class MKLDNNDeviceContext : public CPUDeviceContext { using ShapeBlob = umap_key_string_t; using BlobMap = umap_value_smart_t; + using ExecMap = std::unordered_map< + void*, std::vector, KeyBlob::iterator>>>; + explicit MKLDNNDeviceContext(CPUPlace place); /* \brief Get the active engine */ const mkldnn::engine& GetEngine() const { return tls().get_engine(); } + // Register object to currently used executor's map + void LinkEntryWithExecutor(BlobPtr_t, KeyBlob::iterator) const; + // Remove all entries from the blob map - void ResetBlobMap(); + void ResetBlobMap(void* ptr); // Prevent next ResetBlobMap() void BlockNextCacheClearing(); @@ -753,6 +762,9 @@ class MKLDNNDeviceContext : public CPUDeviceContext { private: std::shared_ptr p_blobmap_; + // Map key is pointer of executor and value is a data(iterator in map) needed + // to erase + std::shared_ptr p_exec_items_; std::shared_ptr p_mutex_; bool block_next_cache_clearing_ = false; }; diff --git a/paddle/fluid/platform/mkldnn_helper.h b/paddle/fluid/platform/mkldnn_helper.h index 35776b9f1e6b8..0b683a742c9fd 100644 --- a/paddle/fluid/platform/mkldnn_helper.h +++ b/paddle/fluid/platform/mkldnn_helper.h @@ -135,13 +135,14 @@ inline mkldnn::memory::desc MKLDNNMemDesc(const std::vector& dims, return mkldnn::memory::desc({dims}, data_type, format); } -inline void ClearMKLDNNCache(const platform::Place& place) { +inline void ClearMKLDNNCache(const platform::Place& place, + void* ptr = nullptr) { // Clear mkl-dnn cache, if (platform::is_cpu_place(place)) { platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); platform::MKLDNNDeviceContext* dev_ctx = (platform::MKLDNNDeviceContext*)pool.Get(place); - dev_ctx->ResetBlobMap(); + dev_ctx->ResetBlobMap(ptr); platform::MKLDNNDeviceContext::tls().set_cur_paddle_data_layout( paddle::framework::DataLayout::kNCHW); } @@ -452,6 +453,9 @@ inline void AttachPointerHashToMKLDNNKey(void* ptr, paddle::platform::MKLDNNDeviceContext::tls().set_key_suffix( "E" + std::to_string(reinterpret_cast(ptr))); } + // Let's register adress of current executor + paddle::platform::MKLDNNDeviceContext::tls().set_curr_exec(ptr); + // For first thread if (first_thread == ThreadIDasStr()) { paddle::platform::MKLDNNDeviceContext::tls().disable_tid_in_key(); From a5627df331b913c047f78789c4c9b3ee5edfe65c Mon Sep 17 00:00:00 2001 From: Wilber Date: Thu, 29 Apr 2021 13:51:18 +0800 Subject: [PATCH 009/156] fix mem release error. (#32655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 后续修复计划是啥 --- .../fluid/inference/api/analysis_predictor.cc | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 6a6be14fd5977..89c8c7902bac9 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -191,22 +191,8 @@ bool AnalysisPredictor::PrepareScope( status_is_cloned_ = true; } else { paddle::framework::InitDevices(); - scope_.reset(new paddle::framework::Scope(), [](framework::Scope *scope) { - delete scope; -#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) - for (int dev_id = 0; dev_id < paddle::platform::GetCUDADeviceCount(); - ++dev_id) { - memory::Release(platform::CUDAPlace(dev_id)); - } -#endif -#ifdef PADDLE_WITH_XPU - for (int dev_id = 0; dev_id < paddle::platform::GetXPUDeviceCount(); - ++dev_id) { - memory::Release(platform::XPUPlace(dev_id)); - } -#endif - memory::Release(platform::CPUPlace()); - }); + // TODO(wilber): we need to release memory occupied by weights. + scope_.reset(new paddle::framework::Scope()); status_is_cloned_ = false; } sub_scope_ = &scope_->NewScope(); From 263710c9cb20bf3ba11d89a9fd79f08672b7a004 Mon Sep 17 00:00:00 2001 From: WeiXin Date: Thu, 29 Apr 2021 13:59:27 +0800 Subject: [PATCH 010/156] edit paddle.save/load API (#32532) (#32612) * edit paddle.save/load API * Update io.py edit doc * delete cpython-37.pyc * Update io.py edit doc * Update io.py recommit * Update io.py recommit * Update io.py recommit * Update io.py recommit --- python/paddle/framework/io.py | 8 ++++---- .../static_mode_white_list.cpython-37.pyc | Bin 20443 -> 0 bytes 2 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 tools/__pycache__/static_mode_white_list.cpython-37.pyc diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index f84ed941e35fe..493574c5bef47 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -496,7 +496,7 @@ def save(obj, path, protocol=2, **configs): Save an object to the specified path. .. note:: - Now supports saving ``state_dict`` of Layer or Optimizer, Tensor. + Now supports saving ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor. .. note:: Different from ``paddle.jit.save``, since the save result of ``paddle.save`` is a single file, @@ -560,7 +560,7 @@ def save(obj, path, protocol=2, **configs): prog = paddle.static.default_main_program() for var in prog.list_vars(): if list(var.shape) == [224, 10]: - tensor = var.get_tensor() + tensor = var.get_value() break # save/load tensor @@ -667,7 +667,7 @@ def load(path, **configs): Load an object can be used in paddle from specified path. .. note:: - Now supports load ``state_dict`` of Layer or Optimizer, Tensor. + Now supports loading ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor. .. note:: In order to use the model parameters saved by paddle more efficiently, @@ -760,7 +760,7 @@ def load(path, **configs): prog = paddle.static.default_main_program() for var in prog.list_vars(): if list(var.shape) == [224, 10]: - tensor = var.get_tensor() + tensor = var.get_value() break # save/load tensor diff --git a/tools/__pycache__/static_mode_white_list.cpython-37.pyc b/tools/__pycache__/static_mode_white_list.cpython-37.pyc deleted file mode 100644 index b1e58ce7689c7db6cc0ce4ed18f87752b16d8beb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20443 zcmeI4XP6{Mm99rxj1UNf7v6&eNMdG$@E!reNPq^{m?XPSR&~-@m8Hz8?rGt@2jRW< zZar&R?;Y!%)-UTl>z8%+Jti`X9Q4<)QuZX-AI85jjOpm2Z%1$Tj6!a&0+Ht|O<*b>(_;hFo86 zAZN-Aic2?*+lZVSA1ygWfJkxS)?@+5h(TqY;vDe_c#nmk>eAzSiHd6qm|o+Ft&S8~~wk%$yB zmWk|0DbJH#sbnfMsbwygOCzl;WGQ>HFDKqI`#Zr+k-uw|tL$uY5_q zPrhG%Kz>kuNPbv;M1E9$OnzK`LVi+yN`6{?Mt)X)PQENZFTWtaD8D4XEWaYZD!(Sb zF25naDObsF$#2W=$nVPU$?wY_$REmAf&*{` zcs_Umcp-QZcrkbhcqw=pcsY0lcqMohcr|zpcrADxcs+Oncq4cdcr&;Xyal`!ybZh^ zyaT)wybHV=ya&7&ybrt|d;ok9dk6-13wOa z0{kTSDe%+aXTZ;bp95b8KM#HZ{37@z@XO#=z^{T|1HTS_1N0^AbZ3fvmp2HY0h4%{Bx0o)PX z3EUan1>6h2TEmzTke~{@?-Nf#5;l z!Qdg_q2OWQ;ouSAk>DcmDDY_T7;rIoEO;DvJa_`Q1Y8Q92%ZF<3@!sFz*E3e!PCIg z!85=XcqVujcs6(r$iQ>La?pSlEWi@%fqifi z9Dpmp^T7+i3&D%Pi@{64OTo*)%fTzaE5WP4tHEo)Yr*Tl>%kkq8^N2vo57XfE#R%- zZQ$+T9pIhdUEtl|J>b3Iec=7z1K@+;L*T>UBjBUpW8mZ96X28JQ{dC!GvKq}b61`^ z`Qp|Qe){}nN6!8NvCY=a7Dc9$NH9vFXoMYvz%q4W~?nYw0EjnvZiPY z`|wSIRPQe5*&^Sr3hj7{P3rde=!M#JhBeJ|A}M{mfjzb?V^K6p@B!Q6@?|j_v1VK1 z){VT|fV#I%o7Z&}eCXaTPR{e0WO=(jtT(JvnUzk#ncfYuoCV!k*3k!(UVEdhN~Yc| zwtI|j^PJX2ok=lY>}a=J`RXoT=1n14bu5#zUF5oa8qc)Gd^B1%`RG6?zS;R`QSRl7 zvYutLSvF~MNnHsei)xW;S7&c7xwb3edOn)uQ+>M4@M%#_OGF_X&yOGDuM02e-=5|t zvwU8L*fE?Ec;7cupZE3A8N}n9sUd=UlT3HzsI0f>>(P>?AQ0Npmcz!^QqCqikVR6jj9^0sPcoN*}9|moX!#NVzir~UFCxL zA}VaEi*T%EHrlD1Y*NgMCSULqQ@XCd-Q=@Lu@&PtlWdBf1Y3gZ&7`dx(~=mhY4QVG zN4DSS2N}#2Id9v-dv?#|WTx$|4z%m@US2KjtuPDd%*;4ewF_fGOl>*RS3^vAXkjL+ znyYlta64bXUJZ*d7KoXyzs#%h3X`W`>~=od-On3|HE!o^Im#MjZI(}q(7x?_G1|#y zbz`5usSUhB%l3|?;rI@F(8iW-7l>esiQ#=I*l=5zvvb(Se0{asu@Wx zM&C0V=4;z6T^synOJQU!%hAB_ZkO}CS?HEd zjUi0Vw_@8x({*^eKADZG(&irXPzp;oxw@@$50dkvF@cP&XT%s}-WVH|0d5tM*pcaf zi8New9UEnow?*hWKiW+F^$~seoNYu|`VrUhk1Bna@(xyOx)m#` zXVSen6SEXdaJ#BUyJ=E2Ydg7M4`%rUHFqB4LH3SfRZa8V(gtaP3b#xvnT5VTzRCN$ zI%XH{XZE();&$UVq;OKtHz9sG?;;-OsPH`2Bc;(ULoe-DJG1-6_W= z1#X%~`1`grosG@Fx+1-ks@{gFdwG*sisLtF409a&%x>yxTw6!_6q?cRE+IvmA#5}r z3{;ZNJ>A7Hwtc06aYF8l%wz1tVw7R0CbRUsoVLq>=^O3oLnwjdi##~4KUTFl5KK;` zR+s=LRDaO8TNJFUK_j_wr zD|hnoH(N6OrhBF|!D5VswctvnBilQw+;=ULzHQz%b!mQ;-3sf7r^KrspiCu(ODzy@ z-aS5zLm=>C!qZ|}H);Vq25=iYCQM1zT8O-Ie9aYj$?Z1zIhUP5TUDFWb|I6Z9bxb4 zSrNuiibX!!=^pm*JfWjf=tDLk0FsD0q|7H-G2Jd?>-IifVxp$pULvL&7R5b*Mor!W zKaO%OipfFP$gYWSsZBGe2X;~F{n@-;Htj@0@?_4>%B@(9TCkeBXSz*t(1H zqfi@%a#n?f00_pKQFPz6*3J4Cs^M*@##)C-v&?3whKv7Nqs1B8dg$1M$riTh%6-q) zUPs)PO~b;Kt%rVKe+SdAt!@+wmb4NKyPgg694Yq(qYVk*Heq>%-G<}+H^E6M~R%I{-6Su9(kz4v=I!DtrkJG8R z*s29S$jGjmh#z{wvCxCUle^&fN%lwsT@8j2?J!ftki5iWhI7tFXmkH^($ zBh|Hxp%FC9!e4CN!`GNbZ;HjTNsaVFQ1*&OgJrd7?*F=+P8aFXNKf*p&3#2s>9z8r zck%G7I51ZE7L0QkH!x{B1rb$2*u8p=IMj17%H} z!&By(41@>u#^r2UG!uB{(|4k};+Tb|>iMkS-~k!s*M!mZ$}Ny3_DnUFwwmTE=UgD^ zrr{uwT&W|N_4crEOz6y1&2_sn2SrozMLAj4%Qmwds28@>qi0c7U@TlZ$NDpSsxJCy znwQb$2ZVXZ^iJOH#EyFBhRAfvt~3$&tx>b%E5j^FVn~yVmJmIp-(OfR$||J4C$te) zjtR&UGnhuPUpKo*0e)S5NT|rTwRk63yJ$?MU8=@Qwk7FYbWjpJaQ8S~&dAj38F7_5 z?_AD zOCDb`pwd#uZX_i(R0Ax6VI-S6Z*{BD4$Q&Upal^sVyTatr5Kq$T8xNVlYY2cYz*Pe z2rgqq#m!xN8X-zIWPe%I*L?1ie6fRmEt+CrXeRlRd_HIP7}?sQS9gX{X3#9M-6aFr zE<9m!{p(s&GBxPotl#Ki>6QmaeF>uvX+tw7=;gf9XiwMf(S_2&QjHtt*s|v%4HcVu z-@5sJb#7EszYsnvy|nq1csENz9%?;kvXnOsLU7!rl5W-}&489sL~w&0g5f4gA5`ge z)~{hR+{fyIX%|Z`5?N?l~Y14B3ez!$0c%L zVrhA$Pr97WwM?Y7FRPY>9uLSu1~D23qXzVGGCke*yOAdO><}U$mJ8`%5KV9}S{5mi zG?F^IdtK?$IUovqFR8euNo7nm~ZT+T*~&&)%RlA7enETkTZS)rQlsBncG&Bly7tBJE=(e`qzk8M>i+Qew>VCC;A z#i1%I>u_IPoeq~6f;uzJB`dSQ)awCdB8uoPr&K65D;nE+ywEt;ShA9qhx23<@||{G zj3NTD5J9Wbsov)?#5!5E)q~G+M)YRxCbX|)HaXg%lA4sGUTP_VN*z(26`?||o6W3r zj6F0v>gKZC>|*ZBtwTxq!2)9Vo+wf1h2j=sx6%Ybi($GuZbNn(+y+m+gz9LwTHjK$ z%2_+DYp>n=OI41)>xQ-3QTJXPhNzuz+LX@{{nOo@$e_EOlJxCvRtD`Pk8pQmMD@Au zrkyhtMUI)4Y%~d`roBUrtwLsEJgT^q$o-PWAs7`d4aT!-pwKG$Ntaw4XnG^>(V#cbjXb~(30r<}5+oiQa2tcBAI z)W%+GB_i#s@4iw(<|D#}Lh*<+A#za?#{Lv55&@l?$IvUS!z7|2W1&|FLR8{|aXeY% z=*yet&4dIJw~u!Zp>shba8Sp^9+D=+^q`S|>Co=F;@Lmlz1EZ`6JPzC(*iqJ*2{>4 zqO{W#Dk8FnT1&W+Czfikq6sW zj4=WI-WbNU0VoMtT5)GqYXlkrKtoFqgM@2jtGkNHMa|T27+SM zBzT^uV;4gz_1siXZC4b@Wr)&=NNE@sQW{JN$_w=2W zPCLVD78$l4>e`uId_a$A$OvN9X2DYh!kL_mx^q>Z@+#EWAL4X~vny5GCQK-YquAjj z&G9j`$^_SWMFlX6>ABb@!Z2Hh=4a5MNcN2QxXD5_jN`=LBy6LUQ^I~fmeX2es7)WW zxOMHG*$~TJ$tp97L;K-?^l+D9VJ-QyCUOyKt{evIA}!-dC$kZKX_&<+2YZxC7^2m1 z7YzUYVO=N!FJ}>UY_^+rva(iBf%QaPHT9UqG@G3Xk|I{{*Yq3;nb9CW%-ZHV(-|Xg zoqIJSGBx%SuGrA2(zE5!!{%T%ir4K3G|Ji5@zrtLHrI%YO6mVJRgbE4^l3pwqT3r^ zO`mpkv#?dJ)fWe0TZBgpXdUB_z-^lb34~o`rDcAX$a#5^O>B3F#i}BteqL7x6Jh}^ zns+0I`upLv>6gp65;zj-56Odo1os`t)K@z|po5z|%Q2(y7?NWL;)F}Hf;>{Q3ez>- zaHFQ>`_gh_t>b&m!4;4e(s(+iM{H5=NLJ1QQjeNaP|;lFwjdq71;6|t4lR*X&PwIK zIl`jJPMj3G5l)RBdurv*eK_<*O}I;ap&g}q1jLghR+P5-LzAw%w|E&oHmXTy9J(Ga z#Mva$Ms{Z)+8(SwFl(v=_gnLH2*Z-5+vUtMxd_JkHVrwdV$b9Piyl_DZ=dUdzj@Ax z5C%<4yxshe??r*LQvLH}pnoZnhb~c%l&J4O9fXiBS(->OhQ+g1%5*AIK+xsP~# zgMz#k4SU$z6l1Mdnyn$C_G=!ouwnvZn}f85VzSh<0#_K!YI=%kqI>10uoFLwUP_J= z=ZJW;jtf^j_-A!SNSB#>3)xg;2;YuiFQ}4eS%9M=J*sg5Y>=YJgMu(^k$Z~_gH`zA zQGp7peTa2Jir{UhsMrggM)c;G&rU&=GfUAKEbS6o8i#{QaW%&inwt*p zaqNJh-RrvhqjrxOaga|Ut)(F?f%pZLkRg0VQhWN})m#9!f%8PaR}-iD_DmESv6MUa z5PZV%a$zB(2Z@#`dIGHRk{-UPBlO93_jG*u_Q(}_f*_`0moxf6&n-u~uI2fLGoaN3 zykk&XOf>R@kj}=u$bjS0OxU-y=N^q_V`F%@vaMA_J8SVEDh?21#z~Z%V+ay)vzHB+ zfm_aYrzz1aJq--Y5*%{JgrU__1iHK6hQHEFdg)@|ImUT5XR5b*R?>RU^m{J5s->>u z8W+;ValxM0V|O=$ES=87;J(#!VP+L8^W2!s*&^Cl*$PoRd#ru+RS4}9t5>1cTRbU5 zjaTQSmVQp9=OzR7)mL3pTI&;OOzeY^=n;D~H#Lk|S{*kla57wdtTEET=&0{`HnT9D@<%V~R4fbQ#DP#LN1UR1gl;wm>ho6?$^KZmv86vcMgSFY#Sw(@x7 za-y}moj2>Ov1{yunIpSu(`}8_#MR#PyT$2!g<&sEtz77ov|g@=xvsZ(HsSVQf$j0D z4TluleTvH&p5NG+p2xMb*Lg%`Su22pBE4@Wbi8W_hCf6Nt;XK9k%e$+RE+_vb&J2$ z1j9wer@x#NqDOUFt(UrXdOJ6Tz zc-JY6>>Up3s!rEHT#S^mcf8Ylw`Tp7;C@n0VV1?C;+ z=~`;s(O1g8tzIy253)husK&a|4ElL)pt#-29DS7r{#APilk;0o$bAqi_Z^WsVx4}r<=@i*R z#4%Lu!E6+p5_j%2SLpKI6!dPSs@&>MLFh&sUepqTFw=*meL5{Kkw?A#f9#Xx+$tas ztoIi?u~s`SPv+G3(dw8~vbo#rHnZ@)FtT5YNZ7IM{@HH)uE$EJviWwCWtIc*Q(6=?ZBIx0D)1CKm z*-%t!7e(j&Qws-D(U3xsxIC}Nh{=bh&Jn_Hk}X7C|7dd0OviK5ZkpKt-R_3B=6G(o zbSieaS!3?&RKC!b!!Z@-Q{L(|K8wW+?>?f?_l%;cn)BFw#%HmabDf;JGkld z4qCEfcWv;qP5g{LycfhV^)XX`zNZ&G4(qwr(JjrP9sAKdpjcr-XrEpqUJhpHGf#)~ z{Rk=TSv!00J4AqX$k}_}C**1EoYmX4A)6cF-??Hgpn8Veo6SuA4%7?qm z!Oi&fj5j`u!@O^jAU@&#DcaVuVPAH%9WtEWSYCUj8&2ESKGhAUZT)b)bVKepqwnBh zr>=9Q!`heo(J6WLFFRWXcdq&--@Y})7`*DDzTUp}+08-pw4ndU<{;X>Hb|#0{d=8a ztaWvxw-)8}ulYTWwPwQBzK2hEJ$~xjJ-S_QKV*oIYt`qqjqdb9s)VMdpl)$={GU@4|)h6&uU-Pbf@2V z>v!N>^Pkvm*MW zI~sknCur^?@VS_O=Q)??iT06Jt#)|9t~aL`T}H5qW9qMQwf=Ud{;TIWR6oOn*=Cms^CWyG*HAa-@WD{#UH0>Cb?zL!o>K9K2ancm`*q=!ZB4!;fT;OIXKc7TS zV@2gY!D9WMI{TeXjq_s+C-|9C3iH#%(-QT!AJZ>1+N+U-ZC)9%U5!M>`MTg&@m9j4 z6pAb|Q$F+RFg{%o(B*kZ*T3(-a5k@&9QemUpAOrL(A9=BUV5`rd|2RGV24%RMV_?d zen-wWAS9*cmls>=Fny5XqlHKWqkD$NZw5w;?ey-_Eyqs#k%zlHz>}fB?PF!;=`5n3 zisIF-l8#-x9kuPWP1&qUCArz2W}lHz%*%P96`+vesD0RP+N_AR<$=f2cy3!>?E^rt__SmMJAYIeG%Hvd*_ReNZxl~fr~OqJqxuG= zLo>bT6W?;4MS~x94~7YMbu9IEarV;*>*oF;9d->#@tf>Ma{qP6C1bC=#!mW8ID5gT z>qj`oc}hU)|5NeX4t9-l$vLN7b=GAk9(Lm5M`Ta9^pO{3Coa0|#Ko69I(z)Zm!0^` zk>;BGPoC_ruU&e6!_~X<^(?wQzbmx+Pe>VBHuWt2{3o(?RdN1;IxIcCGOu&VD7b0m zm7~J*<_A}u_CSc-j(hN#`ahD|;Y+8U^7TLeKXc$3r+od-*Ew)F2UIV#CH>p~lqlYQ M_}gEn4mt3D0KTbuYybcN From 93d34f835b91af1bf94229626210afea789b7c48 Mon Sep 17 00:00:00 2001 From: WeiXin Date: Thu, 29 Apr 2021 13:59:48 +0800 Subject: [PATCH 011/156] 'jit.save/load' support save/load function without parameters. (#32430) (#32613) * jit.save/load support function. * delete unnittest test_jit_load_model_incomplete. * edit code according to CI * Modify the documentation. * add note to doc. --- python/paddle/fluid/dygraph/io.py | 4 + python/paddle/fluid/dygraph/jit.py | 180 +++++++++++------- .../tests/unittests/test_jit_save_load.py | 66 ++++++- 3 files changed, 177 insertions(+), 73 deletions(-) diff --git a/python/paddle/fluid/dygraph/io.py b/python/paddle/fluid/dygraph/io.py index ce40fde1630ad..33eb16f1b2b44 100644 --- a/python/paddle/fluid/dygraph/io.py +++ b/python/paddle/fluid/dygraph/io.py @@ -650,6 +650,7 @@ def _construct_params_and_buffers(model_path, append_suffix=True): var_info_filename = str(params_filename) + ".info" var_info_path = os.path.join(model_path, var_info_filename) + params_path = os.path.join(model_path, str(params_filename)) if os.path.exists(var_info_path): var_dict = _load_persistable_vars(model_path, var_info_path, @@ -671,6 +672,9 @@ def _construct_params_and_buffers(model_path, var_dict.update( _load_persistable_vars(model_path, var_info_path, programs[ func_name], file_name)) + elif params_filename is not None and not os.path.exists(params_path): + # When saving XX, there is only '*.pdmodel' + return dict() else: var_dict = _load_persistable_vars_by_program( model_path, programs['forward'], params_filename) diff --git a/python/paddle/fluid/dygraph/jit.py b/python/paddle/fluid/dygraph/jit.py index 4c7c7b17eb1c4..352a377fa3adc 100644 --- a/python/paddle/fluid/dygraph/jit.py +++ b/python/paddle/fluid/dygraph/jit.py @@ -19,6 +19,7 @@ import warnings import functools from collections import OrderedDict +import inspect import six import paddle @@ -506,7 +507,7 @@ def _build_load_path_and_config(path, config): @switch_to_static_graph def save(layer, path, input_spec=None, **configs): """ - Saves input Layer as ``paddle.jit.TranslatedLayer`` + Saves input Layer or function as ``paddle.jit.TranslatedLayer`` format model, which can be used for inference or fine-tuning after loading. It will save the translated program and all related persistable @@ -522,8 +523,12 @@ def save(layer, path, input_spec=None, **configs): - ``paddle.static.load_inference_model`` - Other C++ inference APIs + .. note:: + When using ``paddle.jit.save`` to save a function, parameters will not be saved. If you have to + save the parameter, please pass the Layer containing function and parameter to ``paddle.jit.save``. + Args: - layer (Layer): The Layer to be saved. + layer (Layer|function): The Layer or function to be saved. path (str): The path prefix to save model. The format is ``dirname/file_prefix`` or ``file_prefix``. input_spec (list[InputSpec|Tensor]|tuple[InputSpec|Tensor], optional): Describes the input of the saved model's forward method, which can be described by InputSpec or example Tensor. If None, all input variables of @@ -543,6 +548,7 @@ def save(layer, path, input_spec=None, **configs): Examples: .. code-block:: python + # example 1: save layer import numpy as np import paddle import paddle.nn as nn @@ -609,6 +615,28 @@ def train(layer, loader, loss_fn, opt): # save path = "example_model/linear" paddle.jit.save(layer, path) + + # example 2: save function + import paddle + from paddle.static import InputSpec + + + def save_function(): + @paddle.jit.to_static + def fun(inputs): + return paddle.tanh(inputs) + + path = 'test_jit_save_load_function_1/func' + inps = paddle.rand([3, 6]) + origin = fun(inps) + + paddle.jit.save(fun, path) + load_func = paddle.jit.load(path) + + load_result = load_func(inps) + print((load_result - origin).abs().max() < 1e-10) + + save_function() """ # 1. input build & check @@ -617,9 +645,11 @@ def train(layer, loader, loss_fn, opt): raise RuntimeError( "The paddle.jit.save doesn't work when setting ProgramTranslator.enable to False." ) - if not isinstance(layer, Layer): + + if not (isinstance(layer, Layer) or inspect.isfunction(layer) or isinstance( + layer, StaticFunction)): raise TypeError( - "The input layer of paddle.jit.save should be 'Layer', but received layer type is %s." + "The input of paddle.jit.save should be 'Layer' or 'Function', but received input type is %s." % type(layer)) # NOTE(chenweihang): If the input layer be wrapped by DataParallel, @@ -647,13 +677,15 @@ def train(layer, loader, loss_fn, opt): # avoid change user given input_spec inner_input_spec = None if input_spec is not None: - for attr_func in dir(inner_layer): - static_func = getattr(inner_layer, attr_func, None) - if isinstance(static_func, - StaticFunction) and 'forward' != attr_func: - raise ValueError( - "If there are static functions other than 'forward' that need to be saved, the input 'input_spec' should be None, but received the type of 'input_spec' is %s." - % type(input_spec)) + if isinstance(layer, Layer): + for attr_func in dir(inner_layer): + static_func = getattr(inner_layer, attr_func, None) + if isinstance(static_func, + StaticFunction) and 'forward' != attr_func: + raise ValueError( + "If there are static functions other than 'forward' that need to be saved, the input 'input_spec' should be None, but received the type of 'input_spec' is %s." + % type(input_spec)) + if not isinstance(input_spec, (list, tuple)): raise TypeError( "The input input_spec should be 'list', but received input_spec's type is %s." @@ -674,29 +706,74 @@ def train(layer, loader, loss_fn, opt): configs = _parse_save_configs(configs) scope = core.Scope() extra_var_info = dict() - for attr_func in dir(inner_layer): - static_func = getattr(inner_layer, attr_func, None) - if isinstance(static_func, StaticFunction): - concrete_program = static_func.concrete_program_specify_input_spec( - inner_input_spec) - elif 'forward' == attr_func: - # transform in jit.save, if input_spec is incomplete, declarative will throw error - # inner_input_spec is list[InputSpec], it should be packed with same sturcture - # as original input_spec here. - if inner_input_spec: - inner_input_spec = pack_sequence_as(input_spec, - inner_input_spec) - static_forward = declarative( - inner_layer.forward, input_spec=inner_input_spec) - concrete_program = static_forward.concrete_program - # the input_spec has been used in declarative, which is equal to - # @declarative with input_spec and jit.save without input_spec, - # avoid needless warning - inner_input_spec = None + if isinstance(layer, Layer): + functions = dir(inner_layer) + else: + # layer is function + functions = [layer, ] + for attr_func in functions: + if isinstance(layer, Layer): + static_func = getattr(inner_layer, attr_func, None) + if isinstance(static_func, StaticFunction): + concrete_program = static_func.concrete_program_specify_input_spec( + inner_input_spec) + elif 'forward' == attr_func: + # transform in jit.save, if input_spec is incomplete, declarative will throw error + # inner_input_spec is list[InputSpec], it should be packed with same sturcture + # as original input_spec here. + if inner_input_spec: + inner_input_spec = pack_sequence_as(input_spec, + inner_input_spec) + static_forward = declarative( + inner_layer.forward, input_spec=inner_input_spec) + concrete_program = static_forward.concrete_program + # the input_spec has been used in declarative, which is equal to + # @declarative with input_spec and jit.save without input_spec, + # avoid needless warning + inner_input_spec = None + else: + continue + + # NOTE(chenweihang): we maintain the mapping of variable name to + # structured name, the buffer variable (non-persistable) + # saved to inference program may not need by dygraph Layer, + # we only record the state_dict variable's structured name + state_names_dict = dict() + for structured_name, var in six.iteritems(inner_layer.state_dict()): + state_names_dict[var.name] = structured_name + + # 3. share parameters from Layer to scope & record var info + for param_or_buffer in concrete_program.parameters: + # share to scope + param_or_buffer_tensor = scope.var( + param_or_buffer.name).get_tensor() + src_tensor = param_or_buffer.value().get_tensor() + param_or_buffer_tensor._share_data_with(src_tensor) + # record var info + if param_or_buffer.name not in extra_var_info: + extra_info_dict = dict() + if param_or_buffer.name in state_names_dict: + extra_info_dict['structured_name'] = state_names_dict[ + param_or_buffer.name] + extra_info_dict[ + 'stop_gradient'] = param_or_buffer.stop_gradient + if isinstance(param_or_buffer, ParamBase): + extra_info_dict['trainable'] = param_or_buffer.trainable + extra_var_info[param_or_buffer.name] = extra_info_dict else: - continue - - # 3. build input & output of save_infernece_model + # When layer is a function + if isinstance(attr_func, StaticFunction): + concrete_program = attr_func.concrete_program_specify_input_spec( + inner_input_spec) + else: + if inner_input_spec: + inner_input_spec = pack_sequence_as(input_spec, + inner_input_spec) + static_function = declarative( + attr_func, input_spec=inner_input_spec) + concrete_program = static_function.concrete_program + + # 4. build input & output of save_infernece_model # NOTE(chenweihang): [ Get input variables name ] # There are two cases, whether to prune the inputs or not # - not prune inputs (recommend): @@ -715,32 +792,6 @@ def train(layer, loader, loss_fn, opt): output_vars = _get_output_vars(concrete_program.outputs, configs.output_spec) - # NOTE(chenweihang): we maintain the mapping of variable name to - # structured name, the buffer variable (non-persistable) - # saved to inference program may not need by dygraph Layer, - # we only record the state_dict variable's structured name - state_names_dict = dict() - for structured_name, var in six.iteritems(inner_layer.state_dict()): - state_names_dict[var.name] = structured_name - - # 4. share parameters from Layer to scope & record var info - for param_or_buffer in concrete_program.parameters: - # share to scope - param_or_buffer_tensor = scope.var(param_or_buffer.name).get_tensor( - ) - src_tensor = param_or_buffer.value().get_tensor() - param_or_buffer_tensor._share_data_with(src_tensor) - # record var info - if param_or_buffer.name not in extra_var_info: - extra_info_dict = dict() - if param_or_buffer.name in state_names_dict: - extra_info_dict['structured_name'] = state_names_dict[ - param_or_buffer.name] - extra_info_dict['stop_gradient'] = param_or_buffer.stop_gradient - if isinstance(param_or_buffer, ParamBase): - extra_info_dict['trainable'] = param_or_buffer.trainable - extra_var_info[param_or_buffer.name] = extra_info_dict - # 5. save inference model from paddle.fluid.io import save_inference_model @@ -748,7 +799,7 @@ def train(layer, loader, loss_fn, opt): model_path = dirname # NOTE(chenweihang): because prefix contains model and params filename, # so we don't support set model_filename & params_filename - if 'forward' == attr_func: + if 'forward' == attr_func or not isinstance(layer, Layer): model_filename = file_prefix + INFER_MODEL_SUFFIX params_filename = file_prefix + INFER_PARAMS_SUFFIX else: @@ -782,10 +833,11 @@ def train(layer, loader, loss_fn, opt): # but we can save these information in `jit.save` without changing the original # storage to improve user experience. So we save extra information into # file `***.pdiparams.info` - with scope_guard(scope): - extra_var_info_path = path + INFER_PARAMS_INFO_SUFFIX - with open(extra_var_info_path, 'wb') as f: - pickle.dump(extra_var_info, f, protocol=2) + if isinstance(layer, Layer) and extra_var_info: + with scope_guard(scope): + extra_var_info_path = path + INFER_PARAMS_INFO_SUFFIX + with open(extra_var_info_path, 'wb') as f: + pickle.dump(extra_var_info, f, protocol=2) @dygraph_only diff --git a/python/paddle/fluid/tests/unittests/test_jit_save_load.py b/python/paddle/fluid/tests/unittests/test_jit_save_load.py index 16adcb8f241ea..eef38182f6edf 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_jit_save_load.py @@ -399,15 +399,6 @@ def test_load_dygraph_no_path(self): with self.assertRaises(ValueError): model_dict, _ = fluid.dygraph.load_dygraph(model_path) - def test_jit_load_model_incomplete(self): - model_path = "test_jit_save_load.remove_variables/model" - self.train_and_save_model(model_path) - # remove `.pdiparams` - var_path = model_path + INFER_PARAMS_SUFFIX - os.remove(var_path) - with self.assertRaises(ValueError): - paddle.jit.load(model_path) - def test_jit_load_no_path(self): path = "test_jit_save_load.no_path/model_path" with self.assertRaises(ValueError): @@ -1164,6 +1155,63 @@ def test_save_load_finetune_load(self): self.assertTrue(float(((result_01 - result_11)).abs().max()) < 1e-5) +class TestJitSaveLoadFunction(unittest.TestCase): + def setUp(self): + paddle.disable_static() + + def test_jit_save_load_static_function(self): + @paddle.jit.to_static + def fun(inputs): + return paddle.tanh(inputs) + + path = 'test_jit_save_load_function_1/func' + inps = paddle.rand([3, 6]) + origin = fun(inps) + + paddle.jit.save(fun, path) + load_func = paddle.jit.load(path) + + load_result = load_func(inps) + self.assertTrue((load_result - origin).abs().max() < 1e-10) + + def test_jit_save_load_function_input_spec(self): + @paddle.jit.to_static(input_spec=[ + InputSpec( + shape=[None, 6], dtype='float32', name='x'), + ]) + def fun(inputs): + return paddle.nn.functional.relu(inputs) + + path = 'test_jit_save_load_function_2/func' + inps = paddle.rand([3, 6]) + origin = fun(inps) + + paddle.jit.save(fun, path) + load_func = paddle.jit.load(path) + load_result = load_func(inps) + self.assertTrue((load_result - origin).abs().max() < 1e-10) + + def test_jit_save_load_function_function(self): + def fun(inputs): + return paddle.tanh(inputs) + + path = 'test_jit_save_load_function_3/func' + inps = paddle.rand([3, 6]) + origin = fun(inps) + + paddle.jit.save( + fun, + path, + input_spec=[ + InputSpec( + shape=[None, 6], dtype='float32', name='x'), + ]) + load_func = paddle.jit.load(path) + + load_result = load_func(inps) + self.assertTrue((load_result - origin).abs().max() < 1e-10) + + class TestJitSaveLoadDataParallel(unittest.TestCase): def verify_inference_correctness(self, layer, path): layer.eval() From ef7b6d557ec397bc96219a9b4345b240f3918d4c Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Thu, 29 Apr 2021 14:01:36 +0800 Subject: [PATCH 012/156] Add fake interface for register_hook in static mode (#32642) (#32660) * add fake interface for hook in static mode * add unittests * fix failed unittests --- python/paddle/fluid/framework.py | 14 +++--- .../fluid/tests/unittests/test_detach.py | 12 +----- .../unittests/test_tensor_register_hook.py | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index a280667d03df4..0e9d756848af4 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -246,11 +246,11 @@ def __impl__(*args, **kwargs): def _fake_interface_only_(func): def __impl__(*args, **kwargs): raise AssertionError( - "'%s' should be called by imperative Varible in imperative mode, please run it in dygraph " - "mode. You can turn off paddle.enable_static() if you are in static mode, or turn off " - "ProgramTranslator if you are using @paddle.jit.to_static. If you have to run ProgramTranslator, " - "please use other API to replace '%s'" % (func.__name__, - func.__name__)) + "'%s' only can be called by `paddle.Tensor` in dynamic graph mode. Suggestions:\n" + " 1. If you are in static graph mode, you can switch to dynamic graph mode by turning off `paddle.enable_static()` or calling `paddle.disable_static()`.\n" + " 2. If you are using `@paddle.jit.to_static`, you can turn off ProgramTranslator by calling `paddle.jit.ProgramTranslator().enable(False)`. " + "If you have to translate dynamic graph to static graph, please use other API to replace '%s'." + % (func.__name__, func.__name__)) return __impl__ @@ -1306,6 +1306,10 @@ def clear_gradient(self): """ pass + @fake_interface_only + def register_hook(self, hook): + pass + def __str__(self): return self._to_readable_code() diff --git a/python/paddle/fluid/tests/unittests/test_detach.py b/python/paddle/fluid/tests/unittests/test_detach.py index 38cdd9b727fc5..5a31418205c32 100644 --- a/python/paddle/fluid/tests/unittests/test_detach.py +++ b/python/paddle/fluid/tests/unittests/test_detach.py @@ -152,18 +152,8 @@ def test_NoDetachSingle_DetachMulti(self): def test_detach_exception(self): x = fluid.layers.data(name="a", shape=[3, 4], dtype='float32') y = fluid.layers.fc(input=x, size=10, bias_attr=True) - try: + with self.assertRaises(AssertionError): y_detach = y.detach() - except Exception as e: - # Here is to check - assert type(e) == AssertionError - assert str(e) == ( - "'detach' should be called by imperative Varible " - "in imperative mode, please run it in dygraph mode. You can " - "turn off paddle.enable_static() if you are in static mode, " - "or turn off ProgramTranslator if you are using " - "@paddle.jit.to_static. If you have to run ProgramTranslator, " - "please use other API to replace 'detach'") class TestInplace(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py index a03e4ae4bd989..52256766fed75 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py @@ -39,6 +39,21 @@ def forward(self, x, hook=None, register=False, remove=False): return ret1, out +class SimpleNetForStatic(nn.Layer): + def __init__(self, in_size, out_size): + super(SimpleNetForStatic, self).__init__() + self.linear1 = nn.Linear(in_size, in_size) + self.linear2 = nn.Linear(in_size, out_size) + + def forward(self, x): + ret1 = self.linear1(x) + ret1.register_hook(lambda grad: grad * 2) + + ret2 = self.linear2(ret1) + out = paddle.mean(ret2, axis=-1) + return out + + class TestTensorRegisterHook(unittest.TestCase): def setUp(self): self.seed = 2021 @@ -451,6 +466,34 @@ def test_register_hook_for_stop_gradient_var(self): with self.assertRaises(RuntimeError): x.register_hook(lambda grad: grad * 2) + def test_register_hook_in_static_mode(self): + paddle.enable_static() + + startup_program = paddle.static.Program() + main_program = paddle.static.Program() + with paddle.static.scope_guard(paddle.static.Scope()): + with paddle.static.program_guard(main_program, startup_program): + x = paddle.static.data( + name='x', shape=[None, self.in_size], dtype='float32') + + net = SimpleNetForStatic(self.in_size, self.out_size) + with self.assertRaises(AssertionError): + out = net(x) + + paddle.disable_static() + + def test_register_hook_in_dy2static_mode(self): + net = SimpleNetForStatic(self.in_size, self.out_size) + jit_net = paddle.jit.to_static( + net, input_spec=[paddle.static.InputSpec([None, self.in_size])]) + + data = np.random.uniform( + size=[self.batch_size, self.in_size]).astype('float32') + data_t = paddle.to_tensor(data) + + with self.assertRaises(AssertionError): + out = jit_net(data_t) + HOOK_INIT_VALUE = 10 HOOK_IS_CALLED = False From 30dfa745c7613604fc3073de90fe4abefcb2cef7 Mon Sep 17 00:00:00 2001 From: Pei Yang Date: Thu, 29 Apr 2021 14:49:37 +0800 Subject: [PATCH 013/156] specify multihead_matmul_fuse_pass_v3 QK path (#32659) (#32668) --- paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc b/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc index 1e8349e878781..57bee20247c96 100644 --- a/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc +++ b/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc @@ -753,7 +753,7 @@ PDNode* MultiHeadMatmulV3Pattern::operator()() { pattern->NewNode(transpose2_0_repr())->assert_is_op("transpose2"); auto* transpose2_0_out_var = pattern->NewNode(transpose2_0_out_repr()) ->assert_is_op_output("transpose2"); - transpose2_0_out_var->AsIntermediate()->assert_is_op_input("matmul"); + transpose2_0_out_var->AsIntermediate()->assert_is_op_input("matmul", "X"); auto* matmul_qk = pattern->NewNode(matmul_qk_repr())->assert_is_op("matmul"); auto* matmul_qk_out_var = @@ -827,7 +827,7 @@ PDNode* MultiHeadMatmulV3Pattern::operator()() { auto* transpose2_1_out_var = pattern->NewNode(transpose2_1_out_repr()) ->assert_is_op_output("transpose2"); transpose2_1_out_var->AsIntermediate()->assert_is_op_input( - "matmul"); // link to matmul qk + "matmul", "Y"); // link to matmul qk // Third path to matmul auto* mul2 = pattern->NewNode(mul2_repr())->assert_is_op("matmul"); From 3c324f043042252aa32cf76f2239ad58a96cecec Mon Sep 17 00:00:00 2001 From: wangna11BD <79366697+wangna11BD@users.noreply.github.com> Date: Thu, 29 Apr 2021 18:24:47 +0800 Subject: [PATCH 014/156] [cherry-pick to 2.1] [Modify spectralnorm #32633] (#32667) --- .../unittests/test_dygraph_spectral_norm.py | 139 ++++++++++++ python/paddle/nn/__init__.py | 2 + python/paddle/nn/utils/__init__.py | 3 +- python/paddle/nn/utils/spectral_norm_hook.py | 210 ++++++++++++++++++ 4 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py create mode 100644 python/paddle/nn/utils/spectral_norm_hook.py diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py b/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py new file mode 100644 index 0000000000000..50903c7d045e3 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py @@ -0,0 +1,139 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +import collections +import paddle +import paddle.nn as nn +from paddle.nn.utils import spectral_norm + + +class TestDygraphSpectralNorm(unittest.TestCase): + def setUp(self): + self.init_test_case() + self.set_data() + + def init_test_case(self): + self.batch_size = 3 + self.data_desc = (['x', [2, 12, 12]], ) + self.n_power_iterations = 1 + self.eps = 1e-12 + self.dim = None + + def set_data(self): + self.data = collections.OrderedDict() + for desc in self.data_desc: + data_name = desc[0] + data_shape = desc[1] + data_value = np.random.random( + size=[self.batch_size] + data_shape).astype('float32') + self.data[data_name] = data_value + + def spectral_normalize(self, weight, u, v, dim, power_iters, eps): + shape = weight.shape + weight_mat = weight.copy() + h = shape[dim] + w = np.prod(shape) // h + if dim != 0: + perm = [dim] + [d for d in range(len(shape)) if d != dim] + weight_mat = weight_mat.transpose(perm) + weight_mat = weight_mat.reshape((h, w)) + + u = u.reshape((h, 1)) + v = v.reshape((w, 1)) + for i in range(power_iters): + v = np.matmul(weight_mat.T, u) + v_norm = np.sqrt((v * v).sum()) + v = v / (v_norm + eps) + u = np.matmul(weight_mat, v) + u_norm = np.sqrt((u * u).sum()) + u = u / (u_norm + eps) + sigma = (u * np.matmul(weight_mat, v)).sum() + return weight / sigma + + def test_check_output(self): + linear = paddle.nn.Conv2D(2, 1, 3) + before_weight = linear.weight.numpy().copy() + if self.dim == None: + if isinstance(linear, (nn.Conv1DTranspose, nn.Conv2DTranspose, + nn.Conv3DTranspose, nn.Linear)): + self.dim = 1 + else: + self.dim = 0 + else: + self.dim = (self.dim + len(before_weight)) % len(before_weight) + + sn = spectral_norm( + linear, + n_power_iterations=self.n_power_iterations, + eps=self.eps, + dim=self.dim) + u = sn.weight_u.numpy().copy() + v = sn.weight_v.numpy().copy() + outputs = [] + for name, data in self.data.items(): + output = linear(paddle.to_tensor(data)) + outputs.append(output.numpy()) + self.actual_outputs = linear.weight.numpy() + + expect_output = self.spectral_normalize( + before_weight, u, v, self.dim, self.n_power_iterations, self.eps) + + for expect, actual in zip(expect_output, self.actual_outputs): + self.assertTrue( + np.allclose( + np.array(actual), np.array(expect), atol=0.001)) + + +class TestDygraphWeightNormCase(TestDygraphSpectralNorm): + def init_test_case(self): + self.batch_size = 2 + self.data_desc = (['x', [2, 3, 3]], ) + self.n_power_iterations = 1 + self.eps = 1e-12 + self.dim = None + + +class TestDygraphWeightNormWithIterations(TestDygraphSpectralNorm): + def init_test_case(self): + self.batch_size = 2 + self.data_desc = (['x', [2, 3, 3]], ) + self.n_power_iterations = 2 + self.eps = 1e-12 + self.dim = None + + +class TestDygraphWeightNormWithDim(TestDygraphSpectralNorm): + def init_test_case(self): + self.batch_size = 2 + self.data_desc = (['x', [2, 3, 3]], ) + self.n_power_iterations = 1 + self.eps = 1e-12 + self.dim = 1 + + +class TestDygraphWeightNormWithEps(TestDygraphSpectralNorm): + def init_test_case(self): + self.batch_size = 2 + self.data_desc = (['x', [2, 3, 3]], ) + self.n_power_iterations = 1 + self.eps = 1e-10 + self.dim = None + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index d2f0063af0d22..817fd50118199 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -126,6 +126,8 @@ from .layer.vision import PixelShuffle # noqa: F401 from .layer.container import LayerDict # noqa: F401 +from .utils.spectral_norm_hook import spectral_norm + # TODO: remove loss, keep it for too many used in unitests from .layer import loss # noqa: F401 from ..fluid.dygraph.layers import Layer # noqa: F401 diff --git a/python/paddle/nn/utils/__init__.py b/python/paddle/nn/utils/__init__.py index bf2573d2cbc2d..b6801cfe3208d 100644 --- a/python/paddle/nn/utils/__init__.py +++ b/python/paddle/nn/utils/__init__.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .spectral_norm_hook import spectral_norm from .weight_norm_hook import weight_norm, remove_weight_norm # noqa: F401 __all__ = [ #noqa - 'weight_norm', 'remove_weight_norm' + 'weight_norm', 'remove_weight_norm', 'spectral_norm' ] diff --git a/python/paddle/nn/utils/spectral_norm_hook.py b/python/paddle/nn/utils/spectral_norm_hook.py new file mode 100644 index 0000000000000..96f7b6a8e7a11 --- /dev/null +++ b/python/paddle/nn/utils/spectral_norm_hook.py @@ -0,0 +1,210 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import numpy as np + +import paddle +from ..layer.conv import Conv1DTranspose, Conv2DTranspose, Conv3DTranspose +from ..layer.common import Linear +from .. import functional as F + +__all__ = [] + + +def normal_(x, mean=0., std=1.): + temp_value = paddle.normal(mean, std, shape=x.shape) + x.set_value(temp_value) + return x + + +class SpectralNorm(object): + def __init__(self, name='weight', n_power_iterations=1, dim=0, eps=1e-12): + self.name = name + self.dim = dim + if n_power_iterations <= 0: + raise ValueError('Expected n_power_iterations to be positive, but ' + 'got n_power_iterations={}'.format( + n_power_iterations)) + self.n_power_iterations = n_power_iterations + self.eps = eps + + def reshape_weight_to_matrix(self, weight): + weight_mat = weight + if self.dim != 0: + # transpose dim to front + weight_mat = weight_mat.transpose([self.dim] + [ + d for d in range(weight_mat.dim()) if d != self.dim + ]) + + height = weight_mat.shape[0] + + return weight_mat.reshape([height, -1]) + + def compute_weight(self, layer, do_power_iteration): + weight = getattr(layer, self.name + '_orig') + u = getattr(layer, self.name + '_u') + v = getattr(layer, self.name + '_v') + weight_mat = self.reshape_weight_to_matrix(weight) + + if do_power_iteration: + with paddle.no_grad(): + for _ in range(self.n_power_iterations): + v.set_value( + F.normalize( + paddle.matmul( + weight_mat, + u, + transpose_x=True, + transpose_y=False), + axis=0, + epsilon=self.eps, )) + + u.set_value( + F.normalize( + paddle.matmul(weight_mat, v), + axis=0, + epsilon=self.eps, )) + if self.n_power_iterations > 0: + u = u.clone() + v = v.clone() + + sigma = paddle.dot(u, paddle.mv(weight_mat, v)) + weight = weight / sigma + return weight + + def __call__(self, layer, inputs): + setattr( + layer, + self.name, + self.compute_weight( + layer, do_power_iteration=layer.training)) + + @staticmethod + def apply(layer, name, n_power_iterations, dim, eps): + for k, hook in layer._forward_pre_hooks.items(): + if isinstance(hook, SpectralNorm) and hook.name == name: + raise RuntimeError("Cannot register two spectral_norm hooks on " + "the same parameter {}".format(name)) + + fn = SpectralNorm(name, n_power_iterations, dim, eps) + weight = layer._parameters[name] + + with paddle.no_grad(): + weight_mat = fn.reshape_weight_to_matrix(weight) + h, w = weight_mat.shape + + # randomly initialize u and v + u = layer.create_parameter([h]) + u = normal_(u, 0., 1.) + v = layer.create_parameter([w]) + v = normal_(v, 0., 1.) + u = F.normalize(u, axis=0, epsilon=fn.eps) + v = F.normalize(v, axis=0, epsilon=fn.eps) + + # delete fn.name form parameters, otherwise you can not set attribute + del layer._parameters[fn.name] + layer.add_parameter(fn.name + "_orig", weight) + # still need to assign weight back as fn.name because all sorts of + # things may assume that it exists, e.g., when initializing weights. + # However, we can't directly assign as it could be an Parameter and + # gets added as a parameter. Instead, we register weight * 1.0 as a plain + # attribute. + setattr(layer, fn.name, weight * 1.0) + layer.register_buffer(fn.name + "_u", u) + layer.register_buffer(fn.name + "_v", v) + layer.register_forward_pre_hook(fn) + return fn + + +def spectral_norm(layer, + name='weight', + n_power_iterations=1, + eps=1e-12, + dim=None): + r""" + This spectral_norm layer applies spectral normalization to a parameter according to the + following Calculation: + + Step 1: + Generate vector U in shape of [H], and V in shape of [W]. + While H is the :attr:`dim` th dimension of the input weights, + and W is the product result of remaining dimensions. + + Step 2: + :attr:`power_iters` should be a positive integer, do following + calculations with U and V for :attr:`power_iters` rounds. + + .. math:: + + \mathbf{v} := \\frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2} + + \mathbf{u} := \\frac{\mathbf{W} \mathbf{v}}{\|\mathbf{W} \mathbf{v}\|_2} + + Step 3: + Calculate :math:`\sigma(\mathbf{W})` and normalize weight values. + + .. math:: + + \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v} + + \mathbf{W} = \\frac{\mathbf{W}}{\sigma(\mathbf{W})} + + + Refer to `Spectral Normalization `_ . + + Parameters: + layer(Layer): Layer of paddle, which has weight. + name(str, optional): Name of the weight parameter. Default: 'weight'. + n_power_iterations(int, optional): The number of power iterations to calculate spectral norm. Default: 1. + eps(float, optional): The epsilon for numerical stability in calculating norms. Default: 1e-12. + dim(int, optional): The index of dimension which should be permuted to the first before reshaping Input(Weight) to matrix, it should be set as 0 if Input(Weight) is the weight of fc layer, and should be set as 1 if Input(Weight) is the weight of conv layer. Default: None. + + Returns: + The original layer with the spectral norm hook + + Examples: + .. code-block:: python + + from paddle.nn import Conv2D + from paddle.nn.utils import Spectralnorm + + conv = Conv2D(3, 1, 3) + sn_conv = spectral_norm(conv) + print(sn_conv) + # Conv2D(3, 1, kernel_size=[3, 3], data_format=NCHW) + print(sn_conv.weight) + # Tensor(shape=[1, 3, 3, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=False, + # [[[[-0.21090528, 0.18563725, -0.14127982], + # [-0.02310637, 0.03197737, 0.34353802], + # [-0.17117859, 0.33152047, -0.28408015]], + # + # [[-0.13336606, -0.01862637, 0.06959272], + # [-0.02236020, -0.27091628, -0.24532901], + # [ 0.27254242, 0.15516677, 0.09036587]], + # + # [[ 0.30169338, -0.28146112, -0.11768346], + # [-0.45765871, -0.12504843, -0.17482486], + # [-0.36866254, -0.19969313, 0.08783543]]]]) + + """ + + if dim is None: + if isinstance(layer, (Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, + Linear)): + dim = 1 + else: + dim = 0 + SpectralNorm.apply(layer, name, n_power_iterations, dim, eps) + return layer From ca2ef4143893972fe71651ba7f422b2b6ddb8236 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Thu, 29 Apr 2021 20:51:57 +0800 Subject: [PATCH 015/156] [Cherry-pick] Polish custom operator overrided method impl (#32666) (#32674) cherry-pick of #32666 --- paddle/fluid/framework/custom_operator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/framework/custom_operator.cc b/paddle/fluid/framework/custom_operator.cc index 97d58df6dc573..c4b833ec94c29 100644 --- a/paddle/fluid/framework/custom_operator.cc +++ b/paddle/fluid/framework/custom_operator.cc @@ -246,7 +246,7 @@ class CustomOperator : public OperatorWithKernel { * it can only be determined at runtime. */ framework::OpKernelType GetExpectedKernelType( - const framework::ExecutionContext& ctx) const { + const framework::ExecutionContext& ctx) const override { return framework::OpKernelType(proto::VarType::RAW, ctx.GetPlace()); } @@ -257,7 +257,7 @@ class CustomOperator : public OperatorWithKernel { */ framework::OpKernelType GetKernelTypeForVar( const std::string& var_name, const Tensor& tensor, - const OpKernelType& expected_kernel_type) { + const OpKernelType& expected_kernel_type) const override { return OpKernelType(expected_kernel_type.data_type_, expected_kernel_type.place_, tensor.layout()); } From 93535c59043a4d6b10d7d982deb928ec98be884d Mon Sep 17 00:00:00 2001 From: arlesniak Date: Thu, 29 Apr 2021 17:19:57 +0200 Subject: [PATCH 016/156] Added pure_bf16 mode (#32281) (#32681) This is cherry-pick of #32281 --- paddle/fluid/operators/assign_op.cc | 1 + .../fluid/contrib/mixed_precision/__init__.py | 3 - .../contrib/mixed_precision/bf16/__init__.py | 4 +- .../contrib/mixed_precision/bf16/amp_lists.py | 14 +- .../contrib/mixed_precision/bf16/amp_utils.py | 219 +++++++++++- .../contrib/mixed_precision/bf16/decorator.py | 318 ++++++++++++++++++ .../fluid/contrib/tests/test_bf16_utils.py | 26 +- .../contrib/tests/test_model_cast_to_bf16.py | 36 +- python/paddle/fluid/layers/nn.py | 3 +- python/paddle/fluid/layers/tensor.py | 7 +- .../fluid/tests/book/test_fit_a_line.py | 78 +++-- .../fluid/tests/book/test_word2vec_book.py | 39 ++- .../tests/unittests/test_optimizer_grad.py | 32 +- python/paddle/static/amp/__init__.py | 5 +- 14 files changed, 699 insertions(+), 86 deletions(-) create mode 100644 python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py diff --git a/paddle/fluid/operators/assign_op.cc b/paddle/fluid/operators/assign_op.cc index add533bafcb0a..433cabcfee010 100644 --- a/paddle/fluid/operators/assign_op.cc +++ b/paddle/fluid/operators/assign_op.cc @@ -162,6 +162,7 @@ REGISTER_OP_CPU_KERNEL_FUNCTOR(assign, float, ops::AssignKernel, double, ops::AssignKernel, int, ops::AssignKernel, int64_t, ops::AssignKernel, bool, ops::AssignKernel, plat::float16, + ops::AssignKernel, plat::bfloat16, ops::AssignKernel); #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) diff --git a/python/paddle/fluid/contrib/mixed_precision/__init__.py b/python/paddle/fluid/contrib/mixed_precision/__init__.py index 571b755b50d2a..a580ae5574c35 100644 --- a/python/paddle/fluid/contrib/mixed_precision/__init__.py +++ b/python/paddle/fluid/contrib/mixed_precision/__init__.py @@ -20,10 +20,7 @@ from .fp16_lists import * from . import fp16_utils from .fp16_utils import * -from . import bf16 -from .bf16 import * __all__ = decorator.__all__ __all__ += fp16_lists.__all__ __all__ += fp16_utils.__all__ -__all__ += bf16.__all__ diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/__init__.py b/python/paddle/fluid/contrib/mixed_precision/bf16/__init__.py index 8c05bc4899cf7..d3632729a3b02 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/__init__.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/__init__.py @@ -18,7 +18,9 @@ from .amp_lists import * from . import amp_utils from .amp_utils import * +from . import decorator +from .decorator import * -__all__ = [] +__all__ = decorator.__all__ __all__ += amp_lists.__all__ __all__ += amp_utils.__all__ diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py index 81dc32d114b14..1cf54aa0838ab 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py @@ -13,8 +13,10 @@ # limitations under the License. import copy +from paddle.fluid import core + from ..fp16_lists import white_list as white_list_fp16, black_list as black_list_fp16,\ - gray_list as gray_list_fp16, unsupported_fp16_list + gray_list as gray_list_fp16 __all__ = ["AutoMixedPrecisionListsBF16"] @@ -82,11 +84,17 @@ def _update_list(self): # depends on the prev_op type gray_list = { + 'cast', + 'fill_constant', + 'reduce_mean', 'reshape2', - 'lookup_table', + 'scale', } -unsupported_list = unsupported_fp16_list.copy().copy() +_, _, _sys_unsupported_bf16_list = core.op_supported_infos( + 'CPU', core.VarDesc.VarType.BF16) +unsupported_list = _sys_unsupported_bf16_list + fp32_list = black_list_fp16.copy().copy() fp32_list |= white_list_fp16 fp32_list |= gray_list_fp16 diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py index c2c01f88c7431..038479098a623 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py @@ -14,18 +14,25 @@ # limitations under the License. from __future__ import print_function -import struct from .... import core from .... import framework +from .... import global_scope from ....log_helper import get_logger from ....wrapped_decorator import signature_safe_contextmanager from .amp_lists import AutoMixedPrecisionListsBF16 -from ..fp16_utils import find_true_prev_op, find_true_post_op, _rename_arg, find_op_index +from ..fp16_utils import find_true_prev_op, find_true_post_op, _rename_arg, \ + find_op_index, _rename_op_input + +import collections +import struct import logging import numpy as np -__all__ = ["bf16_guard", "rewrite_program_bf16", "convert_float_to_uint16"] +__all__ = [ + "bf16_guard", "rewrite_program_bf16", "cast_model_to_bf16", + "cast_parameters_to_bf16", "convert_float_to_uint16" +] _logger = get_logger( __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s') @@ -126,7 +133,41 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): return num_cast_ops +def _insert_cast_post_op(block, op, idx, src_dtype, dest_dtype, target_name, + op_var_rename_map): + num_cast_ops = 0 + target_var = block.var(target_name) + if target_var.type not in _valid_types or target_var.dtype == dest_dtype: + return num_cast_ops + + assert target_var.dtype == src_dtype, \ + "The real dtype({}) is not equal to the src dtype({})".format(_dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype)) + + cast_name = target_var.name + '.cast_' + _dtype_to_str(dest_dtype) + cast_var = block.vars.get(cast_name) + if cast_var is None or cast_var.dtype != dest_dtype: + cast_var = block.create_var( + name=cast_name, + dtype=dest_dtype, + persistable=False, + stop_gradient=target_var.stop_gradient) + block._insert_op( + idx, + type="cast", + inputs={"X": target_var}, + outputs={"Out": cast_var}, + attrs={"in_dtype": target_var.dtype, + "out_dtype": cast_var.dtype}) + num_cast_ops += 1 + op_var_rename_map[block.idx][target_var.name] = cast_var.name + + return num_cast_ops + + def _is_in_fp32_varnames(op, amp_lists): + if not amp_lists.fp32_varnames: + return False + for in_name in op.input_arg_names: if in_name in amp_lists.fp32_varnames: return True @@ -191,7 +232,174 @@ def bf16_guard(): yield -def rewrite_program_bf16(main_prog, amp_lists=None, use_bf16_guard=False): +def cast_model_to_bf16(program, amp_lists=None, use_bf16_guard=True): + """ + Traverse all ops in the whole model and set their inputs and outputs + to the bf16 data type. This function will do some special processing for + the batch normalization, which will keep the batchnorm's computations in FP32. + Args: + program (Program): The used program. + amp_lists (AutoMixedPrecisionListsBF16): An AutoMixedPrecisionListsBF16 object. + use_bf16_guard(bool): Determine whether to use `bf16_guard` when + constructing the program. Default True. + """ + + if amp_lists is None: + amp_lists = AutoMixedPrecisionListsBF16() + global_block = program.global_block() + keep_fp32_ops = set() + to_bf16_var_names = set() + to_bf16_pre_cast_ops = set() + origin_ops = [] + for block in program.blocks: + origin_ops.extend(block.ops) + + for block in program.blocks: + ops = block.ops + for op in ops: + if op.type == 'create_py_reader' or op.type == 'read': + continue + if _need_keep_fp32(op, amp_lists.unsupported_list, use_bf16_guard): + keep_fp32_ops.add(op) + continue # processed below + for in_name in op.input_names: + if op.type in { + 'batch_norm', 'fused_bn_add_activation', 'layer_norm' + } and in_name not in {'X', 'Z'}: + continue + for in_var_name in op.input(in_name): + in_var = None + try: + in_var = block.var(in_var_name) + except ValueError as e: + _logger.debug( + "-- {}, try to get it in the global block --". + format(e)) + in_var = global_block.var(in_var_name) + if in_var is not None: + _logger.debug( + "-- var {} is got in the global block --". + format(in_var_name)) + + if in_var is None or in_var.type not in _valid_types: + continue + + if in_var.dtype == core.VarDesc.VarType.FP32: + in_var.desc.set_dtype(core.VarDesc.VarType.BF16) + to_bf16_var_names.add(in_var_name) + + _logger.debug( + "-- op type: {}, in var name: {}, in var dtype: {} --". + format(op.type, in_var_name, in_var.dtype)) + + for out_name in op.output_names: + if op.type in { + 'batch_norm', 'fused_bn_add_activation', 'layer_norm' + } and out_name != 'Y': + continue + for out_var_name in op.output(out_name): + out_var = None + try: + out_var = block.var(out_var_name) + except ValueError as e: + _logger.debug( + "-- {}, try to get it in the global block --". + format(e)) + out_var = global_block.var(out_var_name) + if out_var is not None: + _logger.debug( + "-- var {} is got in the global block --". + format(out_var_name)) + + if out_var is None or out_var.type not in _valid_types: + continue + + if out_var.dtype == core.VarDesc.VarType.FP32: + out_var.desc.set_dtype(core.VarDesc.VarType.BF16) + + _logger.debug( + "-- op type: {}, out var name: {}, out var dtype: {} --". + format(op.type, out_var_name, out_var.dtype)) + for attr_name in ['in_dtype', 'out_dtype', 'dtype']: + if op.has_attr(attr_name) and op.attr( + attr_name) == core.VarDesc.VarType.FP32: + op._set_attr(attr_name, core.VarDesc.VarType.BF16) + if op.has_attr('use_mkldnn'): + op._set_attr('use_mkldnn', True) + if op.has_attr('mkldnn_data_type'): + op._set_attr('mkldnn_data_type', 'bfloat16') + + # process ops in keep_fp32_ops + op_var_rename_map = [ + collections.OrderedDict() for _ in range(len(program.blocks)) + ] + for block in program.blocks: + ops = block.ops + idx = 0 + while idx < len(ops): + op = ops[idx] + num_cast_ops = 0 + if op not in keep_fp32_ops: + if op in to_bf16_pre_cast_ops: + in_var_cast_num = _insert_cast_op(block, op, idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.BF16) + num_cast_ops += in_var_cast_num + else: + pre_cast_num = _insert_cast_op(block, op, idx, + core.VarDesc.VarType.BF16, + core.VarDesc.VarType.FP32) + num_cast_ops += pre_cast_num + for out_var_name in op.output_arg_names: + out_var = block.vars.get(out_var_name) + if out_var is None or out_var.type not in _valid_types: + continue + if out_var.dtype == core.VarDesc.VarType.BF16: + out_var.desc.set_dtype(core.VarDesc.VarType.FP32) + post_ops = find_true_post_op(ops, op, out_var_name) + for post_op in post_ops: + if post_op in keep_fp32_ops: + continue + post_cast_num = _insert_cast_post_op( + block, op, idx + pre_cast_num + 1, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.BF16, out_var_name, + op_var_rename_map) + num_cast_ops += post_cast_num + idx += num_cast_ops + 1 + + _rename_op_input(program, op_var_rename_map, origin_ops, keep_fp32_ops) + return to_bf16_var_names + + +def cast_parameters_to_bf16(place, program, scope=None, to_bf16_var_names=None): + """ + Traverse all parameters in the whole model and set them to the BF16 data type. + Whereas, this function will keep parameters of batchnorms in FP32. + Args: + place(fluid.CPUPlace|fluid.CUDAPlace): `place` is used to restore the BF16 weight tensors. + program (Program): The used program. + scope(fluid.Scope, optional): `scope` is used to get the FP32 weight tensor values. + Default is None. + to_bf16_var_names(set|list, optional): The data types of vars in `to_bf16_var_names` + will be set to BF16. Usually, it is the returned + value of `cast_model_to_bf16` API. + """ + all_parameters = [] + for block in program.blocks: + all_parameters.extend(block.all_parameters()) + + bf16_var_names = to_bf16_var_names if to_bf16_var_names else set() + var_scope = scope if scope else global_scope() + for param in all_parameters: + if param.name in bf16_var_names: + _logger.debug("---- cast {} to bf16 dtype ----".format(param.name)) + param_t = var_scope.find_var(param.name).get_tensor() + data = np.array(param_t) + param_t.set(convert_float_to_uint16(data), place) + + +def rewrite_program_bf16(main_prog, amp_lists=None): """ Traverse all ops in current block and insert cast op according to which set current op belongs to. @@ -231,8 +439,7 @@ def rewrite_program_bf16(main_prog, amp_lists=None, use_bf16_guard=False): fp32_op_set.add(op) continue - if op.type in amp_lists.fp32_list or _need_keep_fp32( - op, amp_lists.unsupported_list, use_bf16_guard): + if op.type in amp_lists.fp32_list: fp32_op_set.add(op) elif op.type in amp_lists.bf16_list: bf16_op_set.add(op) diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py new file mode 100644 index 0000000000000..86b5a5df75db0 --- /dev/null +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py @@ -0,0 +1,318 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.fluid import (core, default_main_program, layers, program_guard, + unique_name) +from .amp_utils import (rewrite_program_bf16, cast_model_to_bf16, + cast_parameters_to_bf16) +from .amp_lists import AutoMixedPrecisionListsBF16 +import types +import warnings + +__all__ = ["decorate_bf16"] + + +class OptimizerWithMixedPrecision(object): + """ + Optimizer with mixed-precision (MP) training. This is a wrapper of a common + optimizer, plus the support of mixed-precision pre-training. The object + of this class almost has the same behavior as the common optimizer, with the + methods `minimize()`, `backward()`, `apply_gradients()` implemented. + Additionally, it enables the MP training automatically, i.e, the creation + and maintenance of master parameters, scaling of loss, etc. + + Args: + optimizer (Optimizer): A common Optimizer object. + amp_lists (CustomOpLists): An CustomOpLists object. + use_pure_bf16(bool): Whether to use the pure bf16 training. + use_bf16_guard(bool): Whether to use `bf16_guard` when constructing the program. + + """ + + def __init__(self, optimizer, amp_lists, use_pure_bf16, use_bf16_guard): + self._optimizer = optimizer + self._amp_lists = amp_lists + self._param_grads = None + self._train_program = None + + self._learning_rate = optimizer._learning_rate + self._learning_rate_map = optimizer._learning_rate_map + self._use_pure_bf16 = use_pure_bf16 + self._use_bf16_guard = use_bf16_guard + self._to_bf16_var_names = None + + def _init_amp_var(self): + # Ensure the data type of learning rate vars is float32 (same as the + # master parameter dtype) + if isinstance(self._optimizer._learning_rate, float): + self._optimizer._learning_rate_map[default_main_program()] = \ + layers.create_global_var( + name=unique_name.generate("learning_rate"), + shape=[1], + value=float(self._optimizer._learning_rate), + dtype='float32', + persistable=True) + + def backward(self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None): + """ + Backward propagation or auto differentiation for gradients' computation. + + Args: + loss (Variable): The loss Variable to minimize. + startup_program (Program|None): The startup Program for initializing + parameters in `parameter_list`. + parameter_list (list|None): A list of Variables to update. + no_grad_set (set|None): A set of Variables should be ignored. + callbacks (list|None): A list of callable objects to run when appending + backward operator for one parameter. + + Returns: + A list of (param, grad), which is a tuple of a parameter and its + gradient respectively, and the scaled loss. + """ + train_program = loss.block.program + self._train_program = train_program + + with program_guard(self._train_program, startup_program): + self._init_amp_var() + + if self._use_pure_bf16: + self._to_bf16_var_names = cast_model_to_bf16( + self._train_program, self._amp_lists, self._use_bf16_guard) + else: + rewrite_program_bf16(self._train_program, self._amp_lists) + + if loss.dtype != core.VarDesc.VarType.FP32: + loss = loss.astype('float32') + + params_grads = self._optimizer.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks) + return params_grads + + def amp_init(self, + place, + scope=None, + test_program=None, + use_bf16_test=False): + """ + Init the amp training, such as cast fp32 parameters to bf16 type. + + Args: + place(CPUPlace): place is used to initialize + bf16 parameters with fp32 values. + scope(Scope): The scope is used to find fp32 parameters. + test_program(Program): The program is used for testing. + use_bf16_test(bool): Whether to use bf16 testing. + + Examples: + .. code-block:: python + + import numpy as np + import paddle + import paddle.nn.functional as F + paddle.enable_static() + + def run_example_code(): + place = paddle.CPUPlace(0) + exe = paddle.static.Executor(place) + data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32') + conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3) + # 1) Use bf16_guard to control the range of bf16 kernels used. + with paddle.static.amp.bf16_guard(): + bn = paddle.static.nn.batch_norm(input=conv2d, act="relu") + pool = F.max_pool2d(bn, kernel_size=2, stride=2) + hidden = paddle.static.nn.fc(pool, size=10) + loss = paddle.mean(hidden) + # 2) Create the optimizer and set `multi_precision` to True. + # Setting `multi_precision` to True can avoid the poor accuracy + # or the slow convergence in a way. + optimizer = paddle.optimizer.Momentum(learning_rate=0.01, multi_precision=True) + # 3) These ops in `custom_fp32_list` will keep in the float32 computation type. + amp_list = paddle.static.amp.CustomOpLists( + custom_fp32_list=['pool2d']) + # 4) The entry of Paddle AMP. + # Enable pure bf16 training by setting `use_pure_bf16` to True. + optimizer = paddle.static.amp.bf16.decorate_bf16( + optimizer, + amp_list, + use_pure_bf16=True) + # If you don't use the default_startup_program(), you sholud pass + # your defined `startup_program` into `minimize`. + optimizer.minimize(loss) + exe.run(paddle.static.default_startup_program()) + # 5) Use `amp_init` after FP32 parameters initialization(such as `exe.run(startup_program)`). + # If you want to perform the testing process, you should pass `test_program` into `amp_init`. + optimizer.amp_init(place, scope=paddle.static.global_scope()) + + """ + assert self._train_program is not None, \ + "Please call the minimize method first." + if self._use_pure_bf16: + cast_parameters_to_bf16(place, self._train_program, scope, + self._to_bf16_var_names) + if test_program is not None: + if self._use_pure_bf16: + cast_model_to_bf16(test_program, self._amp_lists, + self._use_bf16_guard) + elif use_bf16_test: + rewrite_program_bf16(test_program, self._amp_lists) + + def apply_gradients(self, params_grads): + """ + Apply gradients. + + Args: + params_grads (list): A list of params. + + Returns: + A list of optimize operators. + """ + + return self._optimizer.apply_gradients(params_grads) + + def apply_optimize(self, loss, startup_program, params_grads): + program = loss.block.program + with program_guard(program, startup_program): + optimize_ops = self.apply_gradients(params_grads) + return optimize_ops + + def minimize(self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None): + """ + Perform optimization by minimizing the given loss. + + Args: + loss (Variable): The loss Variable. + startup_program (Program): startup_program for initializing parameters + in `parameter_list`. + parameter_list (list): list of Variables to update. + no_grad_set (set|None): set of Variables should be ignored. + + Returns: + The scaled loss by scaling factor, the list of optimize ops, and a + list of scaled parameters and gradients. + """ + opt_dict = self._optimizer.__class__.__dict__ + if 'minimize' in opt_dict and isinstance(opt_dict['minimize'], + types.FunctionType): + warnings.warn( + "The decorated optimizer has its own `minimize` method, but it will not be executed." + ) + + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set) + + optimize_ops = self.apply_optimize(loss, startup_program, params_grads) + + return optimize_ops, params_grads + + +def decorate_bf16(optimizer, + amp_lists=None, + use_pure_bf16=False, + use_bf16_guard=None): + """ + Decorate the given optimizer to adapt to the mixed-precision training. + + Args: + optimizer(Optimizer): A common Optimizer. + amp_lists (CustomOpLists): An CustomOpLists object. + use_pure_bf16(bool): Whether to use the pure bf16 training. Default False. + use_bf16_guard(bool): Whether to use `bf16_guard` when constructing the program. + Default None, which means that its value equals to `use_pure_bf16`. + + Returns: + An optimizer acting like a normal one but with mixed-precision training + enabled. + + Examples 1: + .. code-block:: python + + # fp32&bf16 list based strategy example + import paddle + import paddle.static as static + + paddle.enable_static() + + data = static.data(name='X', shape=[None, 1], dtype='float32') + hidden = static.nn.fc(x=data, size=10) + loss = paddle.mean(hidden) + optimizer = paddle.optimizer.Adam(learning_rate=0.001) + + mp_optimizer = static.amp.decorate_bf16(optimizer=optimizer) + + ops, param_grads = mp_optimizer.minimize(loss) + + Examples 2: + .. code-block:: python + + # pure bf16 training example + import numpy as np + import paddle + import paddle.nn.functional as F + + def run_example_code(): + place = paddle.CPUPlace(0) + exe = paddle.static.Executor(place) + data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32') + conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3) + # 1) Use bf16_guard to control the range of bf16 kernels used. + with paddle.static.amp.bf16_guard(): + bn = paddle.static.nn.batch_norm(input=conv2d, act="relu") + pool = F.max_pool2d(bn, kernel_size=2, stride=2) + hidden = paddle.static.nn.fc(pool, size=10) + loss = paddle.mean(hidden) + # 2) Create the optimizer and set `multi_precision` to True. + # Setting `multi_precision` to True can avoid the poor accuracy + # or the slow convergence in a way. + optimizer = paddle.optimizer.Momentum(learning_rate=0.01, multi_precision=True) + # 3) These ops in `custom_fp32_list` will keep in the float32 computation type. + amp_list = paddle.static.amp.CustomOpLists( + custom_fp32_list=['pool2d']) + # 4) The entry of Paddle AMP. + # Enable pure bf16 training by setting `use_pure_bf16` to True. + optimizer = paddle.static.amp.decorate_bf16( + optimizer, + amp_list, + use_pure_bf16=True) + # If you don't use the default_startup_program(), you sholud pass + # your defined `startup_program` into `minimize`. + optimizer.minimize(loss) + exe.run(paddle.static.default_startup_program()) + # 5) Use `amp_init` after FP32 parameters initialization(such as `exe.run(startup_program)`). + # If you want to perform the testing process, you should pass `test_program` into `amp_init`. + optimizer.amp_init(place, scope=paddle.static.global_scope()) + + """ + if amp_lists is None: + amp_lists = AutoMixedPrecisionListsBF16() + + if use_bf16_guard is None: + use_bf16_guard = use_pure_bf16 + + mp_optimizer = OptimizerWithMixedPrecision(optimizer, amp_lists, + use_pure_bf16, use_bf16_guard) + + return mp_optimizer diff --git a/python/paddle/fluid/contrib/tests/test_bf16_utils.py b/python/paddle/fluid/contrib/tests/test_bf16_utils.py index faf2307f8147b..2969b7ea11d21 100644 --- a/python/paddle/fluid/contrib/tests/test_bf16_utils.py +++ b/python/paddle/fluid/contrib/tests/test_bf16_utils.py @@ -14,7 +14,7 @@ import copy import unittest import paddle.fluid as fluid -import paddle.fluid.contrib.mixed_precision as amp +import paddle.static.amp as amp from paddle.fluid import core import paddle @@ -34,34 +34,34 @@ def tearDown(self): self.assertEqual(self.amp_lists_.gray_list, self.gray_list) def test_amp_lists(self): - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16() + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16() def test_amp_lists_1(self): # 1. w={'exp}, b=None self.bf16_list.add('exp') self.fp32_list.remove('exp') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16({'exp'}) + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16({'exp'}) def test_amp_lists_2(self): # 2. w={'tanh'}, b=None self.fp32_list.remove('tanh') self.bf16_list.add('tanh') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16({'tanh'}) + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16({'tanh'}) def test_amp_lists_3(self): # 3. w={'lstm'}, b=None self.bf16_list.add('lstm') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16({'lstm'}) + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16({'lstm'}) def test_amp_lists_4(self): # 4. w=None, b={'elementwise_add'} self.bf16_list.remove('elementwise_add') self.fp32_list.add('elementwise_add') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16( + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_list={'elementwise_add'}) def test_amp_lists_5(self): @@ -69,28 +69,28 @@ def test_amp_lists_5(self): self.fp32_list.add('elementwise_add') self.bf16_list.remove('elementwise_add') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16( + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_list={'elementwise_add'}) def test_amp_lists_6(self): # 6. w=None, b={'lstm'} self.fp32_list.add('lstm') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16( + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_list={'lstm'}) def test_amp_lists_7(self): self.fp32_list.add('reshape2') self.gray_list.remove('reshape2') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16( + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_list={'reshape2'}) def test_amp_list_8(self): self.bf16_list.add('reshape2') self.gray_list.remove('reshape2') - self.amp_lists_ = amp.AutoMixedPrecisionListsBF16( + self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( custom_bf16_list={'reshape2'}) @@ -98,7 +98,7 @@ class AMPTest2(unittest.TestCase): def test_amp_lists_(self): # 7. w={'lstm'} b={'lstm'} # raise ValueError - self.assertRaises(ValueError, amp.AutoMixedPrecisionListsBF16, + self.assertRaises(ValueError, amp.bf16.AutoMixedPrecisionListsBF16, {'lstm'}, {'lstm'}) def test_find_op_index(self): @@ -117,10 +117,10 @@ def test_is_in_fp32_varnames(self): type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]}) op2 = block.append_op( type="abs", inputs={"X": [var2]}, outputs={"Out": [var3]}) - amp_lists_1 = amp.AutoMixedPrecisionListsBF16( + amp_lists_1 = amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_varnames={'X'}) assert amp.bf16.amp_utils._is_in_fp32_varnames(op1, amp_lists_1) - amp_lists_2 = amp.AutoMixedPrecisionListsBF16( + amp_lists_2 = amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_varnames={'Y'}) assert amp.bf16.amp_utils._is_in_fp32_varnames(op2, amp_lists_2) assert amp.bf16.amp_utils._is_in_fp32_varnames(op1, amp_lists_2) diff --git a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py index 40ddcf2e66b75..af2c42d6b85ea 100644 --- a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py +++ b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py @@ -65,13 +65,13 @@ def get_static_graph_result(self, feed, fetch_list, amp_fun, fetch_list=fetch_list, return_numpy=(not with_lod)) - def test_graph_rewrite(self): + def _graph_common(self, _amp_fun): size = 3 n = np.ones([size, size], dtype='float32') * 3.2 nn = np.ones([size, size], dtype='float32') * -2.7 - n_bf16 = amp.convert_float_to_uint16(n) - nn_bf16 = amp.convert_float_to_uint16(nn) + n_bf16 = amp.bf16.convert_float_to_uint16(n) + nn_bf16 = amp.bf16.convert_float_to_uint16(nn) with self.static_graph(): t_bf16 = layers.data( @@ -85,12 +85,12 @@ def test_graph_rewrite(self): ret = layers.elementwise_mul(ret, t) ret = layers.reshape(ret, [0, 0]) - with amp.bf16_guard(): + with amp.bf16.bf16_guard(): ret_bf16 = layers.elementwise_add(t_bf16, tt_bf16) ret_bf16 = layers.elementwise_mul(ret_bf16, t_bf16) ret_bf16 = layers.reshape(ret_bf16, [0, 0]) - with amp.bf16_guard(): + with amp.bf16.bf16_guard(): ret_fp32bf16 = layers.elementwise_add(t, tt) ret_fp32bf16 = layers.elementwise_mul(ret_fp32bf16, t) ret_fp32bf16 = layers.reshape(ret_fp32bf16, [0, 0]) @@ -103,7 +103,7 @@ def test_graph_rewrite(self): 'tt_bf16': nn_bf16, }, fetch_list=[ret_bf16, ret, ret_fp32bf16], - amp_fun=lambda prog: amp.rewrite_program_bf16(prog, use_bf16_guard=True)) + amp_fun=lambda prog: amp.bf16.rewrite_program_bf16(prog)) self.assertTrue(np.allclose(static_ret_bf16, static_ret, 1e-2)) self.assertTrue(np.allclose(static_ret_bf16, ret_fp32bf16, 1e-2)) @@ -112,7 +112,7 @@ def test_graph_rewrite(self): t = layers.data(name='t', shape=[size, size], dtype='float32') tt = layers.data(name='tt', shape=[size, size], dtype='float32') - with amp.bf16_guard(): + with amp.bf16.bf16_guard(): ret = layers.elementwise_add(t, tt) ret = layers.reshape(ret, [0, 0], act='elu') ret = layers.elementwise_mul(ret, t) @@ -122,17 +122,27 @@ def test_graph_rewrite(self): self.get_static_graph_result( feed={'t': n, 'tt': nn}, fetch_list=[ret], - amp_fun=lambda prog: amp.rewrite_program_bf16( - prog, - amp.AutoMixedPrecisionListsBF16( - custom_fp32_varnames={'elementwise_add_0.tmp_0'}), - use_bf16_guard=True - ) + amp_fun=_amp_fun ) self.assertTrue( static_ret_bf16, np.ones( [size, size], dtype='float32') * -1.1) + def test_graph_rewrite(self): + self._graph_common(lambda prog: amp.bf16.rewrite_program_bf16( + prog, + amp.bf16.AutoMixedPrecisionListsBF16( + custom_fp32_varnames={'elementwise_add_0.tmp_0'}), + )) + + def test_graph_cast(self): + self._graph_common(lambda prog: amp.bf16.cast_model_to_bf16( + prog, + amp.bf16.AutoMixedPrecisionListsBF16( + custom_fp32_list={'elementwise_mul'}), + use_bf16_guard=True + )) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index e5663d607aa88..751b6251565f5 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -332,7 +332,8 @@ def fc(input, for i, input_x in enumerate(input): check_type(input_x, 'input[' + str(i) + ']', Variable, 'fc') dtype = helper.input_dtype() - check_dtype(dtype, 'input', ['float16', 'float32', 'float64'], 'fc') + check_dtype(dtype, 'input', ['float16', 'uint16', 'float32', 'float64'], + 'fc') mul_results = [] for input_var, param_attr in helper.iter_inputs_and_params(): input_shape = input_var.shape diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index a7ec339bf741e..7dcce5efcfc65 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -582,10 +582,9 @@ def assign(input, output=None): input = numpy.array(input) if isinstance(input, Variable): - check_dtype( - input.dtype, 'input', - ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - 'assign', '(When the type of input in assign is Variable.)') + check_dtype(input.dtype, 'input', [ + 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', 'bool' + ], 'assign', '(When the type of input in assign is Variable.)') if output is None: output = helper.create_variable_for_type_inference( dtype=input.dtype) diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index df43d9366ff78..1172ae0f0ea42 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -16,6 +16,8 @@ import paddle import paddle.fluid as fluid +import paddle.static.amp as amp + import contextlib import numpy import unittest @@ -26,19 +28,34 @@ paddle.enable_static() -def train(use_cuda, save_dirname, is_local, use_bf16): +def train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16): x = fluid.layers.data(name='x', shape=[13], dtype='float32') - - y_predict = fluid.layers.fc(input=x, size=1, act=None) - y = fluid.layers.data(name='y', shape=[1], dtype='float32') - cost = fluid.layers.square_error_cost(input=y_predict, label=y) - avg_cost = fluid.layers.mean(cost) + if use_bf16: + if not pure_bf16: + with amp.bf16.bf16_guard(): + y_predict = fluid.layers.fc(input=x, size=1, act=None) + cost = fluid.layers.square_error_cost(input=y_predict, label=y) + avg_cost = fluid.layers.mean(cost) + else: + y_predict = fluid.layers.fc(input=x, size=1, act=None) + with amp.bf16.bf16_guard(): + cost = fluid.layers.square_error_cost(input=y_predict, label=y) + avg_cost = fluid.layers.mean(cost) + else: + y_predict = fluid.layers.fc(input=x, size=1, act=None) + cost = fluid.layers.square_error_cost(input=y_predict, label=y) + avg_cost = fluid.layers.mean(cost) sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) + if use_bf16: - paddle.static.amp.rewrite_program_bf16(fluid.default_main_program()) + sgd_optimizer = amp.bf16.decorate_bf16( + sgd_optimizer, + amp_lists=amp.bf16.AutoMixedPrecisionListsBF16(), + use_bf16_guard=False, + use_pure_bf16=pure_bf16) sgd_optimizer.minimize(avg_cost) BATCH_SIZE = 20 @@ -54,6 +71,10 @@ def train(use_cuda, save_dirname, is_local, use_bf16): def train_loop(main_program): feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe.run(fluid.default_startup_program()) + test_prog = main_program.clone(for_test=True) + if pure_bf16: + sgd_optimizer.amp_init( + exe.place, test_program=test_prog, use_bf16_test=True) PASS_NUM = 100 for pass_id in range(PASS_NUM): @@ -61,9 +82,8 @@ def train_loop(main_program): avg_loss_value, = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) - print(avg_loss_value) - if avg_loss_value[0] < 10.0: - if save_dirname is not None: + if avg_loss_value[0] < 10.0 or pure_bf16: + if save_dirname is not None and not pure_bf16: fluid.io.save_inference_model(save_dirname, ['x'], [y_predict], exe) return @@ -97,7 +117,7 @@ def train_loop(main_program): train_loop(t.get_trainer_program()) -def infer(use_cuda, save_dirname=None): +def infer(use_cuda, save_dirname=None, use_bf16=False): if save_dirname is None: return @@ -135,7 +155,7 @@ def infer(use_cuda, save_dirname=None): print("ground truth: ", test_label) -def main(use_cuda, is_local=True, use_bf16=False): +def main(use_cuda, is_local=True, use_bf16=False, pure_bf16=False): if use_cuda and not fluid.core.is_compiled_with_cuda(): return @@ -145,11 +165,22 @@ def main(use_cuda, is_local=True, use_bf16=False): # Directory for saving the trained model save_dirname = "fit_a_line.inference.model" - train(use_cuda, save_dirname, is_local, use_bf16) - infer(use_cuda, save_dirname) + train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16) + infer(use_cuda, save_dirname, use_bf16) + + +class TestFitALineBase(unittest.TestCase): + @contextlib.contextmanager + def program_scope_guard(self): + prog = fluid.Program() + startup_prog = fluid.Program() + scope = fluid.core.Scope() + with fluid.scope_guard(scope): + with fluid.program_guard(prog, startup_prog): + yield -class TestFitALine(unittest.TestCase): +class TestFitALine(TestFitALineBase): def test_cpu(self): with self.program_scope_guard(): main(use_cuda=False) @@ -158,20 +189,17 @@ def test_cuda(self): with self.program_scope_guard(): main(use_cuda=True) - @unittest.skipIf(not fluid.core.supports_bfloat16(), - "place does not support BF16 evaluation") + +@unittest.skipIf(not fluid.core.supports_bfloat16(), + "place does not support BF16 evaluation") +class TestFitALineBF16(TestFitALineBase): def test_bf16(self): with self.program_scope_guard(): main(use_cuda=False, use_bf16=True) - @contextlib.contextmanager - def program_scope_guard(self): - prog = fluid.Program() - startup_prog = fluid.Program() - scope = fluid.core.Scope() - with fluid.scope_guard(scope): - with fluid.program_guard(prog, startup_prog): - yield + def test_pure_bf16(self): + with self.program_scope_guard(): + main(use_cuda=False, use_bf16=True, pure_bf16=True) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/book/test_word2vec_book.py b/python/paddle/fluid/tests/book/test_word2vec_book.py index ad7550fa9dd96..f16592a55cf8a 100644 --- a/python/paddle/fluid/tests/book/test_word2vec_book.py +++ b/python/paddle/fluid/tests/book/test_word2vec_book.py @@ -44,7 +44,8 @@ def train(target, is_parallel, save_dirname, is_local=True, - use_bf16=False): + use_bf16=False, + pure_bf16=False): PASS_NUM = 100 EMBED_SIZE = 32 HIDDEN_SIZE = 256 @@ -107,7 +108,13 @@ def __network__(words): sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) if use_bf16: - paddle.static.amp.rewrite_program_bf16(fluid.default_main_program()) + sgd_optimizer = paddle.static.amp.bf16.decorate_bf16( + sgd_optimizer, + amp_lists=paddle.static.amp.bf16.AutoMixedPrecisionListsBF16( + custom_fp32_list={'softmax', 'concat'}, ), + use_bf16_guard=False, + use_pure_bf16=pure_bf16) + sgd_optimizer.minimize(avg_cost) train_reader = paddle.batch( @@ -121,6 +128,8 @@ def __network__(words): def train_loop(main_program): exe.run(fluid.default_startup_program()) + if pure_bf16: + sgd_optimizer.amp_init(exe.place) for pass_id in range(PASS_NUM): for data in train_reader(): @@ -128,7 +137,7 @@ def train_loop(main_program): feed=feeder.feed(data), fetch_list=[avg_cost]) if avg_cost_np[0] < 5.0: - if save_dirname is not None: + if save_dirname is not None and not pure_bf16: fluid.io.save_inference_model(save_dirname, [ 'firstw', 'secondw', 'thirdw', 'forthw' ], [predict_word], exe) @@ -246,7 +255,7 @@ def to_infer_tensor(lod_tensor): assert np.isclose(a, b, rtol=5e-5), "a: {}, b: {}".format(a, b) -def main(target, is_sparse, is_parallel, use_bf16): +def main(target, is_sparse, is_parallel, use_bf16, pure_bf16): if target == "cuda" and not fluid.core.is_compiled_with_cuda(): return if target == "xpu" and not fluid.core.is_compiled_with_xpu(): @@ -265,7 +274,13 @@ def main(target, is_sparse, is_parallel, use_bf16): # so only inference is turned on. train("cpu", is_sparse, is_parallel, save_dirname) else: - train(target, is_sparse, is_parallel, save_dirname, use_bf16=use_bf16) + train( + target, + is_sparse, + is_parallel, + save_dirname, + use_bf16=use_bf16, + pure_bf16=pure_bf16) infer(target, save_dirname) @@ -278,10 +293,15 @@ class W2VTest(unittest.TestCase): pass -def inject_test_method(target, is_sparse, is_parallel, use_bf16=False): +def inject_test_method(target, + is_sparse, + is_parallel, + use_bf16=False, + pure_bf16=False): fn_name = "test_{0}_{1}_{2}{3}".format(target, "sparse" if is_sparse else "dense", "parallel" - if is_parallel else "normal", "_bf16" + if is_parallel else "normal", + "_purebf16" if pure_bf16 else "_bf16" if use_bf16 else "") def __impl__(*args, **kwargs): @@ -290,7 +310,7 @@ def __impl__(*args, **kwargs): scope = fluid.core.Scope() with fluid.scope_guard(scope): with fluid.program_guard(prog, startup_prog): - main(target, is_sparse, is_parallel, use_bf16) + main(target, is_sparse, is_parallel, use_bf16, pure_bf16) if (not fluid.core.is_compiled_with_cuda() or target == "cuda") and is_sparse: @@ -307,7 +327,8 @@ def __impl__(*args, **kwargs): for is_sparse in (False, True): for is_parallel in (False, ): inject_test_method(target, is_sparse, is_parallel) -inject_test_method("cpu", False, False, use_bf16=True) +inject_test_method("cpu", False, False, True) +inject_test_method("cpu", False, False, True, True) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_grad.py b/python/paddle/fluid/tests/unittests/test_optimizer_grad.py index 69298f0f6a55d..7caae211b7bba 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_grad.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_grad.py @@ -64,7 +64,7 @@ def _calc_gradient(self, cond_i): return grads - def build_net(self, cond_i): + def build_net(self, cond_i, use_bf16=False): """ pseudo code: sum_xy = x + y @@ -122,13 +122,22 @@ def cond_false(): sum_cond = fluid.layers.cond(cond_i > 1.0, cond_true, cond_false) sum_all = fluid.layers.sum([sum_xy, sub_yz, sum_cond]) mean_out = fluid.layers.mean(sum_all) + if use_bf16: + import paddle.static.amp as amp + self.optimizer = amp.bf16.decorate_bf16( + self.optimizer, + amp_lists=amp.bf16.AutoMixedPrecisionListsBF16( + custom_fp32_list={'elementwise_add'}), + use_bf16_guard=False, + use_pure_bf16=True) + self.optimizer.minimize(mean_out) fetch_list = ["param_x", "param_z"] if self.y_no_grad else [ "param_x", "param_y", "param_z" ] fetch_list += [_append_grad_suffix_(param) for param in fetch_list] - return fetch_list + return fetch_list, self.optimizer class TestOptimizer(unittest.TestCase): @@ -180,7 +189,7 @@ def _init_param_attr(self): for key in ['x', 'y', 'z']: self.param_attr[key] = self.attr.copy() - def _check_grads(self): + def _check_grads(self, use_bf16=False): """ main logic code to check the validity of apply_optimize. """ @@ -204,10 +213,16 @@ def _check_grads(self): lambda: dict()) test_net = self.NetClass(self.optimizer, param_lr, y_no_grad) - fetch_list = test_net.build_net(cond_i) + fetch_list, decorated_optimizer = test_net.build_net( + cond_i, use_bf16) + if use_bf16: + self.optimizer = decorated_optimizer exe = fluid.Executor(place) exe.run(init_program) + if use_bf16: + self.optimizer.amp_init(exe.place) + # Train 2 steps to check validity for batch_i in range(2): @@ -222,6 +237,15 @@ def _check_grads(self): param_grads[i]) +@unittest.skipIf(not fluid.core.supports_bfloat16(), + "place does not support BF16 evaluation") +class TestSGDOptimizer(TestOptimizer): + def test_optimizer_multiblock_except(self): + with self.assertRaisesRegexp(ValueError, + "var param_y not in this block"): + self._check_grads(use_bf16=True) + + class TestAdamOptimizer(TestOptimizer): """ inherit TestOptimizer and shall override two functions as follows: diff --git a/python/paddle/static/amp/__init__.py b/python/paddle/static/amp/__init__.py index 54de11401f3c6..4c309edfeafe0 100644 --- a/python/paddle/static/amp/__init__.py +++ b/python/paddle/static/amp/__init__.py @@ -18,9 +18,6 @@ from ...fluid.contrib.mixed_precision import fp16_guard # noqa: F401 from ...fluid.contrib.mixed_precision import cast_model_to_fp16 # noqa: F401 from ...fluid.contrib.mixed_precision import cast_parameters_to_fp16 # noqa: F401 -from ...fluid.contrib.mixed_precision import AutoMixedPrecisionListsBF16 # noqa: F401 -from ...fluid.contrib.mixed_precision import bf16_guard # noqa: F401 -from ...fluid.contrib.mixed_precision import rewrite_program_bf16 # noqa: F401 -from ...fluid.contrib.mixed_precision import convert_float_to_uint16 # noqa: F401 +from ...fluid.contrib.mixed_precision import bf16 # noqa: F401 __all__ = [] From e7c8160050e815016453f0a171097dc5d79e5d7a Mon Sep 17 00:00:00 2001 From: "joanna.wozna.intel" Date: Thu, 29 Apr 2021 17:20:53 +0200 Subject: [PATCH 017/156] Add BF16 uniform random initializer (#32468) (#32677) * Add bf16 uniform random initializer * Remove duplicated section * Change UT to CPU place only * Put detail functions into anonymous namespace --- paddle/fluid/operators/fill_constant_op.h | 3 + paddle/fluid/operators/uniform_random_op.cc | 58 +++- paddle/fluid/operators/uniform_random_op.h | 9 +- python/paddle/fluid/initializer.py | 16 +- python/paddle/fluid/layers/nn.py | 7 +- .../fluid/tests/unittests/test_initializer.py | 45 +-- .../tests/unittests/test_initializer_nn.py | 11 +- .../unittests/test_uniform_random_bf16_op.py | 276 ++++++++++++++++++ tools/static_mode_white_list.py | 1 + 9 files changed, 371 insertions(+), 55 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py diff --git a/paddle/fluid/operators/fill_constant_op.h b/paddle/fluid/operators/fill_constant_op.h index 46c4ae12036a4..17c7321122b17 100644 --- a/paddle/fluid/operators/fill_constant_op.h +++ b/paddle/fluid/operators/fill_constant_op.h @@ -117,6 +117,9 @@ class FillConstantKernel : public framework::OpKernel { } if (actual_place == 0) { + VLOG(4) << "[CPU] FillConstantKernel" + << ((data_type == framework::proto::VarType::BF16) ? "" + : ""); tensor->mutable_data(platform::CPUPlace(), data_type); math::SetConstant functor; functor(reinterpret_cast(dev_ctx), diff --git a/paddle/fluid/operators/uniform_random_op.cc b/paddle/fluid/operators/uniform_random_op.cc index 6efada4343ca5..007276b16d7f2 100644 --- a/paddle/fluid/operators/uniform_random_op.cc +++ b/paddle/fluid/operators/uniform_random_op.cc @@ -18,10 +18,41 @@ limitations under the License. */ #include "paddle/fluid/framework/generator.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/platform/bfloat16.h" namespace paddle { namespace operators { +namespace { +template +inline void UniformRealDistribution(T *data, const int64_t &size, + const float &min, const float &max, + const unsigned int &seed) { + VLOG(4) << "[CPU] UniformRandomKernel"; + std::uniform_real_distribution dist(static_cast(min), + static_cast(max)); + auto engine = paddle::framework::GetCPURandomEngine(seed); + + for (int64_t i = 0; i < size; ++i) { + data[i] = dist(*engine); + } +} + +template <> +inline void UniformRealDistribution(paddle::platform::bfloat16 *data, + const int64_t &size, const float &min, + const float &max, + const unsigned int &seed) { + VLOG(4) << "[CPU] UniformRandomKernel"; + std::uniform_real_distribution dist(min, max); + auto engine = paddle::framework::GetCPURandomEngine(seed); + + for (int64_t i = 0; i < size; ++i) { + data[i] = static_cast(dist(*engine)); + } +} +} // namespace + // It seems that Eigen::Tensor::random in GPU will SEGFAULT. // Use std::random and thrust::random(thrust is a std library in CUDA) to // implement uniform random. @@ -61,17 +92,11 @@ class CPUUniformRandomKernel : public framework::OpKernel { framework::ToTypeName(out_var->Type()))); } T *data = tensor->mutable_data(ctx.GetPlace()); - int64_t size = tensor->numel(); - std::uniform_real_distribution dist( - static_cast(ctx.Attr("min")), - static_cast(ctx.Attr("max"))); - unsigned int seed = static_cast(ctx.Attr("seed")); - auto engine = framework::GetCPURandomEngine(seed); - for (int64_t i = 0; i < size; ++i) { - data[i] = dist(*engine); - } + UniformRealDistribution( + data, size, ctx.Attr("min"), ctx.Attr("max"), + static_cast(ctx.Attr("seed"))); unsigned int diag_num = static_cast(ctx.Attr("diag_num")); @@ -257,9 +282,12 @@ REGISTER_OPERATOR( paddle::framework::EmptyGradOpMaker, paddle::operators::UniformRandomOpVarTypeInference); -REGISTER_OP_CPU_KERNEL(uniform_random, - paddle::operators::CPUUniformRandomKernel, - paddle::operators::CPUUniformRandomKernel); -REGISTER_OP_CPU_KERNEL(uniform_random_batch_size_like, - paddle::operators::CPUUniformRandomKernel, - paddle::operators::CPUUniformRandomKernel); +REGISTER_OP_CPU_KERNEL( + uniform_random, paddle::operators::CPUUniformRandomKernel, + paddle::operators::CPUUniformRandomKernel, + paddle::operators::CPUUniformRandomKernel); +REGISTER_OP_CPU_KERNEL( + uniform_random_batch_size_like, + paddle::operators::CPUUniformRandomKernel, + paddle::operators::CPUUniformRandomKernel, + paddle::operators::CPUUniformRandomKernel); diff --git a/paddle/fluid/operators/uniform_random_op.h b/paddle/fluid/operators/uniform_random_op.h index 6052e533643f3..18a4154be30ac 100644 --- a/paddle/fluid/operators/uniform_random_op.h +++ b/paddle/fluid/operators/uniform_random_op.h @@ -24,9 +24,9 @@ namespace operators { using Tensor = framework::Tensor; inline std::vector GetNewDataFromShapeTensor( - const Tensor *new_data_tensor) { + const Tensor* new_data_tensor) { if (new_data_tensor->type() == framework::proto::VarType::INT64) { - auto *new_data = new_data_tensor->data(); + auto* new_data = new_data_tensor->data(); framework::Tensor cpu_starts_tensor; if (platform::is_gpu_place(new_data_tensor->place())) { TensorCopySync(*new_data_tensor, platform::CPUPlace(), @@ -37,7 +37,7 @@ inline std::vector GetNewDataFromShapeTensor( new_data + new_data_tensor->numel()); return vec_new_data; } else if (new_data_tensor->type() == framework::proto::VarType::INT32) { - auto *new_data = new_data_tensor->data(); + auto* new_data = new_data_tensor->data(); std::vector vec_new_data; framework::Tensor cpu_starts_tensor; if (platform::is_gpu_place(new_data_tensor->place())) { @@ -58,7 +58,7 @@ inline std::vector GetNewDataFromShapeTensor( } inline std::vector GetNewDataFromShapeTensorList( - const std::vector &list_new_shape_tensor) { + const std::vector& list_new_shape_tensor) { std::vector vec_new_shape; vec_new_shape.reserve(list_new_shape_tensor.size()); for (size_t i = 0; i < list_new_shape_tensor.size(); ++i) { @@ -97,6 +97,5 @@ inline std::vector GetNewDataFromShapeTensorList( return vec_new_shape; } - } // namespace operators } // namespace paddle diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index dc153614fcd26..5b2010f340958 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -245,7 +245,7 @@ def __call__(self, var, block=None): self._seed = block.program.random_seed # to be compatible of fp16 initializers - if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: + if var.dtype == VarDesc.VarType.FP16: out_dtype = VarDesc.VarType.FP32 out_var = block.create_var( name=unique_name.generate(".".join( @@ -274,7 +274,7 @@ def __call__(self, var, block=None): }, stop_gradient=True) - if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: + if var.dtype == VarDesc.VarType.FP16: block.append_op( type="cast", inputs={"X": out_var}, @@ -540,7 +540,8 @@ def __call__(self, var, block=None): self._seed = block.program.random_seed # to be compatible of fp16 initalizers - if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: + if var.dtype == VarDesc.VarType.FP16 or ( + var.dtype == VarDesc.VarType.BF16 and not self._uniform): out_dtype = VarDesc.VarType.FP32 out_var = block.create_var( name=unique_name.generate(".".join( @@ -582,7 +583,8 @@ def __call__(self, var, block=None): }, stop_gradient=True) - if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: + if var.dtype == VarDesc.VarType.FP16 or ( + var.dtype == VarDesc.VarType.BF16 and not self._uniform): block.append_op( type="cast", inputs={"X": out_var}, @@ -671,7 +673,8 @@ def __call__(self, var, block=None): self._seed = block.program.random_seed # to be compatible of fp16 initalizers - if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: + if var.dtype == VarDesc.VarType.FP16 or ( + var.dtype == VarDesc.VarType.BF16 and not self._uniform): out_dtype = VarDesc.VarType.FP32 out_var = block.create_var( name=unique_name.generate(".".join( @@ -713,7 +716,8 @@ def __call__(self, var, block=None): }, stop_gradient=True) - if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: + if var.dtype == VarDesc.VarType.FP16 or ( + var.dtype == VarDesc.VarType.BF16 and not self._uniform): block.append_op( type="cast", inputs={"X": out_var}, diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 751b6251565f5..9ac314528dc1f 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -10524,10 +10524,10 @@ def uniform_random_batch_size_like(input, """ - check_variable_and_dtype(input, 'Input', ("float32", 'float64'), + check_variable_and_dtype(input, 'Input', ("float32", 'float64', "uint16"), 'uniform_random_batch_size_like') check_type(shape, 'shape', (list, tuple), 'uniform_random_batch_size_like') - check_dtype(dtype, 'dtype', ('float32', 'float64'), + check_dtype(dtype, 'dtype', ('float32', 'float64', "uint16"), 'uniform_random_batch_size_like') helper = LayerHelper('uniform_random_batch_size_like', **locals()) @@ -15121,7 +15121,8 @@ def uniform_random(shape, dtype='float32', min=-1.0, max=1.0, seed=0, float(max), 'seed', seed, 'dtype', dtype) check_type(shape, 'shape', (list, tuple, Variable), 'uniform_random/rand') - check_dtype(dtype, 'dtype', ('float32', 'float64'), 'uniform_random/rand') + check_dtype(dtype, 'dtype', ('float32', 'float64', 'uint16'), + 'uniform_random/rand') inputs = dict() attrs = {'seed': seed, 'min': min, 'max': max, 'dtype': dtype} diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index 237ff0c958e39..8ddb74989714c 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -53,7 +53,7 @@ def test_constant_initializer_default_value(self, dtype="float32"): lod_level=0, name="param", initializer=initializer.ConstantInitializer()) - num_ops = 2 if dtype in ["float16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'fill_constant') @@ -72,7 +72,7 @@ def test_constant_initializer(self, dtype="float32"): lod_level=0, name="param", initializer=initializer.ConstantInitializer(2.3)) - num_ops = 2 if dtype in ["float16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'fill_constant') @@ -108,7 +108,7 @@ def test_uniform_initializer_default_value(self, dtype="float32"): lod_level=0, name="param", initializer=initializer.UniformInitializer()) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -153,7 +153,7 @@ def test_uniform_initializer(self, dtype="float32"): lod_level=0, name="param", initializer=initializer.UniformInitializer(-4.2, 3.1, 123)) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -174,7 +174,7 @@ def test_uniform_initializer_two_op(self, dtype="float32"): lod_level=0, name="param", initializer=initializer.UniformInitializer(-4.2, float(i), 123)) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op0 = block.ops[0] self.assertEqual(init_op0.type, 'uniform_random') @@ -195,13 +195,11 @@ def test_uniform_initializer_fp16(self): def test_uniform_initializer_bf16(self): """Test uniform initializer with bfloat16 + No cast operator has been added here """ block = self.test_uniform_initializer_default_value("uint16") - self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer(dtype="uint16") - self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer_two_op("uint16") - self.assertTrue(check_cast_op(block.ops[1])) class TestNormalInitializer(unittest.TestCase): @@ -347,7 +345,9 @@ def test_normal_xavier_initializer_conv(self): self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) - def test_xavier_initializer_supplied_arguments(self, dtype="float32"): + def test_xavier_initializer_supplied_arguments(self, + dtype="float32", + uniform=True): """Test the Xavier initializer with supplied arguments """ program = framework.Program() @@ -359,14 +359,18 @@ def test_xavier_initializer_supplied_arguments(self, dtype="float32"): lod_level=0, name="param", initializer=initializer.XavierInitializer( - fan_in=12, fan_out=23, seed=134)) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + uniform=uniform, fan_in=12, fan_out=23, seed=134)) + num_ops = 2 if (dtype == "float16" or (dtype == "uint16" and + not uniform)) else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] - self.assertEqual(init_op.type, 'uniform_random') - limit = np.sqrt(6.0 / (12 + 23)) - self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) - self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + if uniform: + self.assertEqual(init_op.type, 'uniform_random') + limit = np.sqrt(6.0 / (12 + 23)) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + else: + self.assertEqual(init_op.type, 'gaussian_random') self.assertEqual(init_op.attr('seed'), 134) return block @@ -379,8 +383,12 @@ def test_xavier_initializer_fp16(self): def test_xavier_initializer_bf16(self): """Test the Xavier initializer with bfloat16 """ - block = self.test_xavier_initializer_supplied_arguments("uint16") - self.assertTrue(check_cast_op(block.ops[1])) + block_uniform = self.test_xavier_initializer_supplied_arguments( + "uint16") + self.assertEqual(len(block_uniform.ops), 1) + block_gaussian = self.test_xavier_initializer_supplied_arguments( + "uint16", False) + self.assertTrue(check_cast_op(block_gaussian.ops[1])) class TestMSRAInitializer(unittest.TestCase): @@ -483,7 +491,7 @@ def test_msra_initializer_supplied_arguments(self, dtype="float32"): name="param", initializer=initializer.MSRAInitializer( fan_in=12, seed=134)) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -503,7 +511,6 @@ def test_msra_initializer_bf16(self): """Test the MSRA initializer with bfloat16 """ block = self.test_msra_initializer_supplied_arguments("uint16") - self.assertTrue(check_cast_op(block.ops[1])) class TestBilinearInitializer(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_initializer_nn.py b/python/paddle/fluid/tests/unittests/test_initializer_nn.py index 9ec78366226f8..85815c5eeef30 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer_nn.py +++ b/python/paddle/fluid/tests/unittests/test_initializer_nn.py @@ -225,7 +225,7 @@ def test_uniform_common(self, dtype="float32", seed=0): lod_level=0, name="param", initializer=initializer.Uniform()) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -256,7 +256,7 @@ def test_uniform_initializer_default_value(self, lod_level=0, name="param", initializer=initializer.Uniform()) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -287,7 +287,7 @@ def test_uniform_initializer(self, lod_level=0, name="param", initializer=initializer.Uniform(min_value, max_vlaue)) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -317,7 +317,7 @@ def test_uniform_initializer_two_op(self, lod_level=0, name="param", initializer=initializer.Uniform(min_value, float(i))) - num_ops = 2 if dtype in ["float16", "uint16"] else 1 + num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op0 = block.ops[0] self.assertEqual(init_op0.type, 'uniform_random') @@ -343,11 +343,8 @@ def test_uniform_initializer_bf16(self): """Test uniform initializer with bfloat16 """ block = self.test_uniform_initializer_default_value("uint16") #bfloat16 - self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer(dtype="uint16") #bfloat16 - self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer_two_op("uint16") #bfloat16 - self.assertTrue(check_cast_op(block.ops[1])) def test_uniform_initializer_dygraph(self): """Test uniform initializer in dygraph model. diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py new file mode 100644 index 0000000000000..2ba808a341e5e --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py @@ -0,0 +1,276 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from op_test import OpTest, convert_uint16_to_float, convert_float_to_uint16 +import paddle +import paddle.fluid.core as core +from paddle.fluid.op import Operator +import paddle.fluid as fluid +from paddle.fluid import Program, program_guard +from paddle.fluid.tests.unittests.test_uniform_random_op import output_hist, output_hist_diag + + +class TestUniformRandomOpBF16(OpTest): + def setUp(self): + self.op_type = "uniform_random" + self.dtype = "uint16" + self.inputs = {} + self.init_attrs() + self.outputs = {"Out": np.zeros((1000, 784)).astype("uint16")} + + def init_attrs(self): + self.attrs = { + "shape": [1000, 784], + "min": -5.0, + "max": 10.0, + "seed": 10, + 'dtype': int(core.VarDesc.VarType.BF16) + } + self.output_hist = output_hist + + def verify_output(self, outs): + if np.array(outs[0]).dtype == np.uint16: + result = convert_uint16_to_float(np.array(outs[0])) + else: + result = np.array(outs[0]) + + hist, prob = self.output_hist(result) + self.assertTrue( + np.allclose( + hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + + def test_check_output(self): + outs = self.calc_output(core.CPUPlace()) + outs = [np.array(out) for out in outs] + outs.sort(key=len) + self.verify_output(outs) + + +class TestUniformRandomOpBF16AttrTensorList(TestUniformRandomOpBF16): + def setUp(self): + self.op_type = "uniform_random" + self.new_shape = (1000, 784) + self.dtype = "uint16" + shape_tensor = [] + for index, ele in enumerate(self.new_shape): + shape_tensor.append(("x" + str(index), np.ones( + (1)).astype("int64") * ele)) + self.inputs = {'ShapeTensorList': shape_tensor} + self.init_attrs() + self.outputs = {"Out": np.zeros((1000, 784)).astype("uint16")} + + def init_attrs(self): + self.attrs = { + "min": -5.0, + "max": 10.0, + "seed": 10, + 'dtype': int(core.VarDesc.VarType.BF16) + } + self.output_hist = output_hist + + +class TestUniformRandomOpBF16AttrTensorInt32( + TestUniformRandomOpBF16AttrTensorList): + def setUp(self): + self.op_type = "uniform_random" + self.dtype = "uint16" + self.inputs = {"ShapeTensor": np.array([1000, 784]).astype("int32")} + self.init_attrs() + self.outputs = {"Out": np.zeros((1000, 784)).astype("uint16")} + + +class TestUniformRandomOpBF16WithDiagInit(TestUniformRandomOpBF16): + def init_attrs(self): + self.attrs = { + "shape": [1000, 784], + "min": -5.0, + "max": 10.0, + "seed": 10, + "diag_num": 784, + "diag_step": 784, + "diag_val": 1.0, + 'dtype': int(core.VarDesc.VarType.BF16) + } + self.output_hist = output_hist_diag + + +class TestUniformRandomOpBF16SelectedRows(unittest.TestCase): + def test_check_output(self): + self.check_with_place(core.CPUPlace()) + + def check_with_place(self, place): + scope = core.Scope() + out = scope.var("X").get_selected_rows() + paddle.seed(10) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16)) + op.run(scope, place) + self.assertEqual(out.get_tensor().shape(), [1000, 784]) + result = convert_uint16_to_float(np.array(out.get_tensor())) + hist, prob = output_hist(result) + self.assertTrue( + np.allclose( + hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + + +class TestUniformRandomOpBF16SelectedRowsWithDiagInit( + TestUniformRandomOpBF16SelectedRows): + def check_with_place(self, place): + scope = core.Scope() + out = scope.var("X").get_selected_rows() + paddle.seed(10) + op = Operator( + "uniform_random", + Out="X", + shape=[500, 784], + min=-5.0, + max=10.0, + seed=10, + diag_num=500, + diag_step=784, + diag_val=1.0, + dtype=int(core.VarDesc.VarType.BF16)) + op.run(scope, place) + self.assertEqual(out.get_tensor().shape(), [500, 784]) + result = convert_uint16_to_float(np.array(out.get_tensor())) + hist, prob = output_hist(result) + self.assertTrue( + np.allclose( + hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + + +class TestUniformRandomOpBF16AttrTensorAPI(unittest.TestCase): + def test_attr_tensor_API(self): + startup_program = fluid.Program() + train_program = fluid.Program() + with fluid.program_guard(train_program, startup_program): + dim_tensor = fluid.layers.fill_constant([1], "int64", 3) + ret = fluid.layers.nn.uniform_random( + [1, dim_tensor, 2], dtype=np.uint16) + + place = fluid.CPUPlace() + exe = fluid.Executor(place) + + exe.run(startup_program) + outs = exe.run(train_program, fetch_list=[ret]) + + +class TestUniformRandomOpAPISeed(unittest.TestCase): + def test_attr_tensor_API(self): + _seed = 10 + gen = paddle.seed(_seed) + gen._is_init_py = False + startup_program = fluid.Program() + train_program = fluid.Program() + with fluid.program_guard(train_program, startup_program): + _min = 5 + _max = 10 + + ret = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed) + ret_2 = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed) + res = fluid.layers.equal(ret, ret_2) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + + exe.run(startup_program) + ret_value, cmp_value = exe.run(train_program, fetch_list=[ret, res]) + self.assertTrue(np.array(cmp_value).all()) + for i in ret_value.flatten(): + self.assertGreaterEqual(i, _min) + self.assertLess(i, _max) + + +class TestUniformRandomOpBF16SelectedRowsShapeTensor(unittest.TestCase): + def test_check_output(self): + place = core.CPUPlace() + scope = core.Scope() + out = scope.var("X").get_selected_rows() + shape_tensor = scope.var("Shape").get_tensor() + shape_tensor.set(np.array([1000, 784]).astype("int64"), place) + paddle.seed(10) + op = Operator( + "uniform_random", + ShapeTensor="Shape", + Out="X", + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16)) + op.run(scope, place) + self.assertEqual(out.get_tensor().shape(), [1000, 784]) + result = convert_uint16_to_float(np.array(out.get_tensor())) + hist, prob = output_hist(result) + self.assertTrue( + np.allclose( + hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + + +class TestUniformRandomOpBF16SelectedRowsShapeTensorList( + TestUniformRandomOpBF16SelectedRowsShapeTensor): + def test_check_output(self): + place = core.CPUPlace() + scope = core.Scope() + out = scope.var("X").get_selected_rows() + shape_1 = scope.var("shape1").get_tensor() + shape_1.set(np.array([1000]).astype("int64"), place) + shape_2 = scope.var("shape2").get_tensor() + shape_2.set(np.array([784]).astype("int64"), place) + paddle.seed(10) + op = Operator( + "uniform_random", + ShapeTensorList=["shape1", "shape2"], + Out="X", + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16)) + op.run(scope, place) + self.assertEqual(out.get_tensor().shape(), [1000, 784]) + result = convert_uint16_to_float(np.array(out.get_tensor())) + hist, prob = output_hist(result) + self.assertTrue( + np.allclose( + hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + + +class TestUniformRandomBatchSizeLikeOpBF16API(unittest.TestCase): + def test_attr_tensorlist_int32_API(self): + startup_program = fluid.Program() + train_program = fluid.Program() + with fluid.program_guard(train_program, startup_program): + input = fluid.data(name="input", shape=[1, 3], dtype='uint16') + out_1 = fluid.layers.uniform_random_batch_size_like( + input, [2, 4], dtype=np.uint16) # out_1.shape=[1, 4] + + place = fluid.CPUPlace() + exe = fluid.Executor(place) + + exe.run(startup_program) + outs = exe.run(train_program, fetch_list=[out_1]) + + +if __name__ == "__main__": + from paddle import enable_static + enable_static() + unittest.main() diff --git a/tools/static_mode_white_list.py b/tools/static_mode_white_list.py index 7c1f54adfb3d9..15bcae826064d 100644 --- a/tools/static_mode_white_list.py +++ b/tools/static_mode_white_list.py @@ -498,6 +498,7 @@ 'test_truncated_gaussian_random_op', 'test_unbind_op', 'test_unfold_op', + 'test_uniform_random_bf16_op', 'test_uniform_random_op', 'test_unique', 'test_unique_with_counts', From cb5065792831331d0c9fd9e79853e0ee917ebe9a Mon Sep 17 00:00:00 2001 From: Shang Zhizhou Date: Fri, 30 Apr 2021 09:21:29 +0800 Subject: [PATCH 018/156] Nne integration (#32604) (#32658) * Add dlnne engine runtime * Remove and remove unrelated modify with dlnne, +clang-format * Add copyright message * Add some paddlepaddle_pass to support more networks * Add delete dropout_op pass Co-authored-by: denglin-github <82362191+denglin-github@users.noreply.github.com> --- paddle/fluid/framework/ir/CMakeLists.txt | 1 + .../framework/ir/delete_dropout_op_pass.cc | 96 +++++++++++++++++++ .../framework/ir/delete_dropout_op_pass.h | 37 +++++++ .../framework/ir/graph_pattern_detector.cc | 23 +++++ .../framework/ir/graph_pattern_detector.h | 13 +++ .../inference/api/paddle_pass_builder.cc | 1 + 6 files changed, 171 insertions(+) create mode 100644 paddle/fluid/framework/ir/delete_dropout_op_pass.cc create mode 100644 paddle/fluid/framework/ir/delete_dropout_op_pass.h diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index 0ca78c679aeca..ab69170322ce3 100644 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -86,6 +86,7 @@ pass_library(quant_conv2d_dequant_fuse_pass inference) pass_library(shuffle_channel_detect_pass inference) pass_library(delete_quant_dequant_op_pass inference) pass_library(delete_quant_dequant_filter_op_pass inference) +pass_library(delete_dropout_op_pass inference) pass_library(simplify_with_basic_ops_pass base) pass_library(fc_elementwise_layernorm_fuse_pass base) pass_library(skip_layernorm_fuse_pass base) diff --git a/paddle/fluid/framework/ir/delete_dropout_op_pass.cc b/paddle/fluid/framework/ir/delete_dropout_op_pass.cc new file mode 100644 index 0000000000000..09962239a01b1 --- /dev/null +++ b/paddle/fluid/framework/ir/delete_dropout_op_pass.cc @@ -0,0 +1,96 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include + +#include "paddle/fluid/framework/ir/delete_dropout_op_pass.h" + +namespace paddle { +namespace framework { +class LoDTensor; +} // namespace framework +} // namespace paddle + +namespace paddle { +namespace framework { +namespace ir { + +#define GET_IR_NODE(node__) GET_IR_NODE_FROM_SUBGRAPH(node__, node__, pattern); +#define GET_NODES \ + GET_IR_NODE(any_op_out); \ + GET_IR_NODE(dropout_op); \ + GET_IR_NODE(dropout_op_out); \ + GET_IR_NODE(dropout_op_outmask); \ + GET_IR_NODE(any_op2); + +void DeleteDropoutOpPass::ApplyImpl(ir::Graph* graph) const { + const std::string pattern_name = "delete_dropout_op_pattern"; + FusePassBase::Init(pattern_name, graph); + + GraphPatternDetector gpd; + + patterns::DeleteDropoutOpPattern pattern(gpd.mutable_pattern(), pattern_name); + pattern(); + + auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, + Graph* g) { + GET_NODES; + IR_NODE_LINK_TO(any_op_out, any_op2); + std::string any_op_out_name = any_op_out->Var()->Name(); + std::string dropout_op_out_name = dropout_op_out->Var()->Name(); + + auto* any_op2_desc = any_op2->Op(); + auto var_map = any_op2_desc->Inputs(); + std::string arg_name = ""; + for (auto& name_m : var_map) { + if (std::find(name_m.second.begin(), name_m.second.end(), + dropout_op_out_name) != name_m.second.end()) { + arg_name = name_m.first; + } + } + if (arg_name.size() == 0) { + LOG(INFO) << "Delete dropout op pass: can not find the input " + << dropout_op_out_name; + return; + } + + // modify the any_op2's inputs + for (auto& name_m : var_map) { + if (std::find(name_m.second.begin(), name_m.second.end(), + dropout_op_out_name) != name_m.second.end()) { + std::vector new_inputs; + for (auto& i_n : name_m.second) { + if (i_n != dropout_op_out_name) { + new_inputs.push_back(i_n); + } + } + new_inputs.push_back(any_op_out_name); + any_op2_desc->SetInput(name_m.first, new_inputs); + any_op2_desc->Flush(); + } + } + any_op2_desc->Flush(); + // Delete the unneeded nodes. + GraphSafeRemoveNodes(graph, + {dropout_op, dropout_op_out, dropout_op_outmask}); + }; + + gpd(graph, handler); +} + +} // namespace ir +} // namespace framework +} // namespace paddle + +REGISTER_PASS(delete_dropout_op_pass, + paddle::framework::ir::DeleteDropoutOpPass); diff --git a/paddle/fluid/framework/ir/delete_dropout_op_pass.h b/paddle/fluid/framework/ir/delete_dropout_op_pass.h new file mode 100644 index 0000000000000..c49abf3c871ce --- /dev/null +++ b/paddle/fluid/framework/ir/delete_dropout_op_pass.h @@ -0,0 +1,37 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include + +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/framework/ir/graph_pattern_detector.h" + +namespace paddle { +namespace framework { +namespace ir { + +class Graph; + +class DeleteDropoutOpPass : public FusePassBase { + public: + virtual ~DeleteDropoutOpPass() {} + + protected: + void ApplyImpl(ir::Graph* graph) const override; +}; + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index d74e8e5f65cd2..064da3d941602 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -2439,6 +2439,29 @@ PDNode *patterns::TransposeFlattenConcat::operator()( return concat_out; } +void patterns::DeleteDropoutOpPattern::operator()() { + auto any_op_out = pattern->NewNode(any_op_out_repr()) + ->assert_is_op_input("dropout", "X") + ->AsInput(); + + auto dropout_op = + pattern->NewNode(dropout_op_repr())->assert_is_op("dropout"); + + auto dropout_op_out = pattern->NewNode(dropout_op_out_repr()) + ->assert_is_op_output("dropout", "Out") + ->AsIntermediate(); + + auto dropout_op_outmask = pattern->NewNode(dropout_op_outmask_repr()) + ->assert_is_op_output("dropout", "Mask") + ->AsOutput(); + auto any_op2 = pattern->NewNode(any_op2_repr())->assert_is_op()->AsOutput(); + + dropout_op->LinksFrom({any_op_out}); + dropout_op_out->LinksFrom({dropout_op}); + dropout_op_outmask->LinksFrom({dropout_op}); + any_op2->LinksFrom({dropout_op_out}); +} + void patterns::DeleteQuantOpFuse::operator()(PDNode *input_act_node, const std::string &quant_type) { auto *input_scale_node = pattern->NewNode(GetNodeName("input_scale_node")) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index cfac01ec9dedc..13f65859954d5 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -1464,6 +1464,19 @@ struct ShuffleChannelPattern : public PatternBase { PATTERN_DECL_NODE(reshape2_out); }; +struct DeleteDropoutOpPattern : public PatternBase { + DeleteDropoutOpPattern(PDPattern* pattern, const std::string& name_scope) + : PatternBase(pattern, name_scope, "delete_dropout_op_pattern") {} + + void operator()(); + + PATTERN_DECL_NODE(any_op_out); + PATTERN_DECL_NODE(dropout_op); + PATTERN_DECL_NODE(dropout_op_out); + PATTERN_DECL_NODE(dropout_op_outmask); + PATTERN_DECL_NODE(any_op2); +}; + struct DeleteQuantDequantOpPattern : public PatternBase { DeleteQuantDequantOpPattern(PDPattern* pattern, const std::string& name_scope) : PatternBase(pattern, name_scope, "delete_quantdequant_op_pattern") {} diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc index 2b7333edae0da..b2e3de63691c5 100644 --- a/paddle/fluid/inference/api/paddle_pass_builder.cc +++ b/paddle/fluid/inference/api/paddle_pass_builder.cc @@ -112,6 +112,7 @@ const std::vector kTRTSubgraphPasses({ const std::vector kDlnneSubgraphPasses({ "is_test_pass", // + "delete_dropout_op_pass" // "simplify_with_basic_ops_pass", // "conv_bn_fuse_pass", // "depthwise_conv_bn_fuse_pass", // From 79ce2a6c31d74500a1b5fcadd9dc6c0d3debf4b6 Mon Sep 17 00:00:00 2001 From: cc <52520497+juncaipeng@users.noreply.github.com> Date: Fri, 30 Apr 2021 09:23:00 +0800 Subject: [PATCH 019/156] skip fuse repeated fc when the fc with weight padding (#32648) (#32680) --- .../framework/ir/repeated_fc_relu_fuse_pass.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc b/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc index 479df876fbe00..bf59c14000516 100644 --- a/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc +++ b/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc @@ -54,6 +54,17 @@ static bool IsFCWithAct(Node* n, const std::string& act_type = "relu") { return false; } +static bool IsFCWithPaddingWeights(Node* n) { + bool res = false; + if (n && n->IsOp() && n->Op() && n->Op()->Type() == "fc" && + n->inputs.size() == 3U && n->outputs.size() == 1U) { + if (n->Op()->HasAttr("padding_weights")) { + res = BOOST_GET_CONST(bool, n->Op()->GetAttr("padding_weights")); + } + } + return res; +} + static bool IsParamOfFC(Node* n, const std::string& param_name) { if (IsInputOfFC(n) && n->inputs.empty() && (n->Name() == n->outputs[0]->Op()->Input(param_name)[0])) { @@ -255,7 +266,7 @@ void BuildRepeatedFCReluPattern(PDPattern* pattern, fc_ops[i] = pattern->NewNode( [=](Node* x) { - if (!IsFCWithAct(x, "relu")) { + if (!IsFCWithAct(x, "relu") || IsFCWithPaddingWeights(x)) { return false; } auto* fc_out_var = x->outputs[0]; From 2817239a639c39ced8a29acd8dfa16b8b0b006f2 Mon Sep 17 00:00:00 2001 From: LielinJiang <50691816+LielinJiang@users.noreply.github.com> Date: Fri, 30 Apr 2021 11:50:20 +0800 Subject: [PATCH 020/156] Add op read_file and decode_jpeg (#32564) (#32686) * add op read_file and decode_jpeg --- cmake/operators.cmake | 1 + paddle/fluid/operators/decode_jpeg_op.cc | 114 +++++++++++++++ paddle/fluid/operators/decode_jpeg_op.cu | 138 ++++++++++++++++++ paddle/fluid/operators/read_file_op.cc | 92 ++++++++++++ paddle/fluid/platform/dynload/CMakeLists.txt | 2 +- .../fluid/platform/dynload/dynamic_loader.cc | 17 +++ .../fluid/platform/dynload/dynamic_loader.h | 1 + paddle/fluid/platform/dynload/nvjpeg.cc | 27 ++++ paddle/fluid/platform/dynload/nvjpeg.h | 53 +++++++ python/paddle/tests/test_read_file.py | 67 +++++++++ python/paddle/vision/ops.py | 97 +++++++++++- 11 files changed, 607 insertions(+), 2 deletions(-) create mode 100644 paddle/fluid/operators/decode_jpeg_op.cc create mode 100644 paddle/fluid/operators/decode_jpeg_op.cu create mode 100644 paddle/fluid/operators/read_file_op.cc create mode 100644 paddle/fluid/platform/dynload/nvjpeg.cc create mode 100644 paddle/fluid/platform/dynload/nvjpeg.h create mode 100644 python/paddle/tests/test_read_file.py diff --git a/cmake/operators.cmake b/cmake/operators.cmake index 7dac91e531e4c..16288e1fb45df 100644 --- a/cmake/operators.cmake +++ b/cmake/operators.cmake @@ -182,6 +182,7 @@ function(op_library TARGET) list(REMOVE_ITEM hip_srcs "cholesky_op.cu") list(REMOVE_ITEM hip_srcs "correlation_op.cu") list(REMOVE_ITEM hip_srcs "multinomial_op.cu") + list(REMOVE_ITEM hip_srcs "decode_jpeg_op.cu") hip_library(${TARGET} SRCS ${cc_srcs} ${hip_cc_srcs} ${miopen_cu_cc_srcs} ${miopen_cu_srcs} ${mkldnn_cc_srcs} ${hip_srcs} DEPS ${op_library_DEPS} ${op_common_deps}) else() diff --git a/paddle/fluid/operators/decode_jpeg_op.cc b/paddle/fluid/operators/decode_jpeg_op.cc new file mode 100644 index 0000000000000..e553b1076a864 --- /dev/null +++ b/paddle/fluid/operators/decode_jpeg_op.cc @@ -0,0 +1,114 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "paddle/fluid/framework/generator.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/platform/dynload/nvjpeg.h" +#include "paddle/fluid/platform/enforce.h" + +namespace paddle { +namespace operators { + +template +class CPUDecodeJpegKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + // TODO(LieLinJiang): add cpu implement. + PADDLE_THROW(platform::errors::Unimplemented( + "DecodeJpeg op only supports GPU now.")); + } +}; + +class DecodeJpegOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "DecodeJpeg"); + OP_INOUT_CHECK(ctx->HasOutput("Out"), "Output", "Out", "DecodeJpeg"); + + auto mode = ctx->Attrs().Get("mode"); + std::vector out_dims; + + if (mode == "unchanged") { + out_dims = {-1, -1, -1}; + } else if (mode == "gray") { + out_dims = {1, -1, -1}; + } else if (mode == "rgb") { + out_dims = {3, -1, -1}; + } else { + PADDLE_THROW(platform::errors::Fatal( + "The provided mode is not supported for JPEG files on GPU: ", mode)); + } + + ctx->SetOutputDim("Out", framework::make_ddim(out_dims)); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace()); + } + + framework::OpKernelType GetKernelTypeForVar( + const std::string& var_name, const framework::Tensor& tensor, + const framework::OpKernelType& expected_kernel_type) const { + if (var_name == "X") { + return expected_kernel_type; + } + + return framework::OpKernelType(tensor.type(), tensor.place(), + tensor.layout()); + } +}; + +class DecodeJpegOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "A one dimensional uint8 tensor containing the raw bytes " + "of the JPEG image. It is a tensor with rank 1."); + AddOutput("Out", "The output tensor of DecodeJpeg op"); + AddComment(R"DOC( +This operator decodes a JPEG image into a 3 dimensional RGB Tensor +or 1 dimensional Gray Tensor. Optionally converts the image to the +desired format. The values of the output tensor are uint8 between 0 +and 255. +)DOC"); + AddAttr( + "mode", + "(string, default \"unchanged\"), The read mode used " + "for optionally converting the image, can be \"unchanged\" " + ",\"gray\" , \"rgb\" .") + .SetDefault("unchanged"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OPERATOR( + decode_jpeg, ops::DecodeJpegOp, ops::DecodeJpegOpMaker, + paddle::framework::EmptyGradOpMaker, + paddle::framework::EmptyGradOpMaker) + +REGISTER_OP_CPU_KERNEL(decode_jpeg, ops::CPUDecodeJpegKernel) diff --git a/paddle/fluid/operators/decode_jpeg_op.cu b/paddle/fluid/operators/decode_jpeg_op.cu new file mode 100644 index 0000000000000..35975a6a54986 --- /dev/null +++ b/paddle/fluid/operators/decode_jpeg_op.cu @@ -0,0 +1,138 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PADDLE_WITH_HIP + +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/platform/dynload/nvjpeg.h" +#include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/stream/cuda_stream.h" + +namespace paddle { +namespace operators { + +static cudaStream_t nvjpeg_stream = nullptr; +static nvjpegHandle_t nvjpeg_handle = nullptr; + +void InitNvjpegImage(nvjpegImage_t* img) { + for (int c = 0; c < NVJPEG_MAX_COMPONENT; c++) { + img->channel[c] = nullptr; + img->pitch[c] = 0; + } +} + +template +class GPUDecodeJpegKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + // Create nvJPEG handle + if (nvjpeg_handle == nullptr) { + nvjpegStatus_t create_status = + platform::dynload::nvjpegCreateSimple(&nvjpeg_handle); + + PADDLE_ENFORCE_EQ(create_status, NVJPEG_STATUS_SUCCESS, + platform::errors::Fatal("nvjpegCreateSimple failed: ", + create_status)); + } + + nvjpegJpegState_t nvjpeg_state; + nvjpegStatus_t state_status = + platform::dynload::nvjpegJpegStateCreate(nvjpeg_handle, &nvjpeg_state); + + PADDLE_ENFORCE_EQ(state_status, NVJPEG_STATUS_SUCCESS, + platform::errors::Fatal("nvjpegJpegStateCreate failed: ", + state_status)); + + int components; + nvjpegChromaSubsampling_t subsampling; + int widths[NVJPEG_MAX_COMPONENT]; + int heights[NVJPEG_MAX_COMPONENT]; + + auto* x = ctx.Input("X"); + auto* x_data = x->data(); + + nvjpegStatus_t info_status = platform::dynload::nvjpegGetImageInfo( + nvjpeg_handle, x_data, (size_t)x->numel(), &components, &subsampling, + widths, heights); + + PADDLE_ENFORCE_EQ( + info_status, NVJPEG_STATUS_SUCCESS, + platform::errors::Fatal("nvjpegGetImageInfo failed: ", info_status)); + + int width = widths[0]; + int height = heights[0]; + + nvjpegOutputFormat_t output_format; + int output_components; + + auto mode = ctx.Attr("mode"); + if (mode == "unchanged") { + if (components == 1) { + output_format = NVJPEG_OUTPUT_Y; + output_components = 1; + } else if (components == 3) { + output_format = NVJPEG_OUTPUT_RGB; + output_components = 3; + } else { + platform::dynload::nvjpegJpegStateDestroy(nvjpeg_state); + PADDLE_THROW(platform::errors::Fatal( + "The provided mode is not supported for JPEG files on GPU")); + } + } else if (mode == "gray") { + output_format = NVJPEG_OUTPUT_Y; + output_components = 1; + } else if (mode == "rgb") { + output_format = NVJPEG_OUTPUT_RGB; + output_components = 3; + } else { + platform::dynload::nvjpegJpegStateDestroy(nvjpeg_state); + PADDLE_THROW(platform::errors::Fatal( + "The provided mode is not supported for JPEG files on GPU")); + } + + nvjpegImage_t out_image; + InitNvjpegImage(&out_image); + + // create nvjpeg stream + if (nvjpeg_stream == nullptr) { + cudaStreamCreateWithFlags(&nvjpeg_stream, cudaStreamNonBlocking); + } + + int sz = widths[0] * heights[0]; + + auto* out = ctx.Output("Out"); + std::vector out_shape = {output_components, height, width}; + out->Resize(framework::make_ddim(out_shape)); + + T* data = out->mutable_data(ctx.GetPlace()); + + for (int c = 0; c < output_components; c++) { + out_image.channel[c] = data + c * sz; + out_image.pitch[c] = width; + } + + nvjpegStatus_t decode_status = platform::dynload::nvjpegDecode( + nvjpeg_handle, nvjpeg_state, x_data, x->numel(), output_format, + &out_image, nvjpeg_stream); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL(decode_jpeg, ops::GPUDecodeJpegKernel) + +#endif diff --git a/paddle/fluid/operators/read_file_op.cc b/paddle/fluid/operators/read_file_op.cc new file mode 100644 index 0000000000000..6da92ed7df7d8 --- /dev/null +++ b/paddle/fluid/operators/read_file_op.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "paddle/fluid/framework/generator.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/platform/enforce.h" + +namespace paddle { +namespace operators { + +template +class CPUReadFileKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto filename = ctx.Attr("filename"); + + std::ifstream input(filename.c_str(), + std::ios::in | std::ios::binary | std::ios::ate); + std::streamsize file_size = input.tellg(); + + input.seekg(0, std::ios::beg); + + auto* out = ctx.Output("Out"); + std::vector out_shape = {file_size}; + out->Resize(framework::make_ddim(out_shape)); + + uint8_t* data = out->mutable_data(ctx.GetPlace()); + + input.read(reinterpret_cast(data), file_size); + } +}; + +class ReadFileOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE_EQ(ctx->HasOutput("Out"), true, + platform::errors::InvalidArgument( + "Output(Out) of ReadFileOp is null.")); + + auto out_dims = std::vector(1, -1); + ctx->SetOutputDim("Out", framework::make_ddim(out_dims)); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType(framework::proto::VarType::UINT8, + platform::CPUPlace()); + } +}; + +class ReadFileOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddOutput("Out", "The output tensor of ReadFile op"); + AddComment(R"DOC( +This operator read a file. +)DOC"); + AddAttr("filename", "Path of the file to be readed.") + .SetDefault({}); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OPERATOR( + read_file, ops::ReadFileOp, ops::ReadFileOpMaker, + paddle::framework::EmptyGradOpMaker, + paddle::framework::EmptyGradOpMaker) + +REGISTER_OP_CPU_KERNEL(read_file, ops::CPUReadFileKernel) diff --git a/paddle/fluid/platform/dynload/CMakeLists.txt b/paddle/fluid/platform/dynload/CMakeLists.txt index b25fb5978d055..8bff2ead0a2a3 100644 --- a/paddle/fluid/platform/dynload/CMakeLists.txt +++ b/paddle/fluid/platform/dynload/CMakeLists.txt @@ -1,6 +1,6 @@ cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags enforce) -list(APPEND CUDA_SRCS cublas.cc cudnn.cc curand.cc cusolver.cc nvtx.cc) +list(APPEND CUDA_SRCS cublas.cc cudnn.cc curand.cc cusolver.cc nvtx.cc nvjpeg.cc) if (WITH_ROCM) list(APPEND HIP_SRCS rocblas.cc miopen.cc hiprand.cc) diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index b49875f256bb2..be9cda4a2e9b6 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -100,6 +100,9 @@ static constexpr char* win_cublas_lib = static constexpr char* win_curand_lib = "curand64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR ".dll;curand64_" CUDA_VERSION_MAJOR ".dll;curand64_10.dll"; +static constexpr char* win_nvjpeg_lib = + "nvjpeg64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR + ".dll;nvjpeg64_" CUDA_VERSION_MAJOR ".dll;nvjpeg64_10.dll"; static constexpr char* win_cusolver_lib = "cusolver64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR ".dll;cusolver64_" CUDA_VERSION_MAJOR ".dll;cusolver64_10.dll"; @@ -107,6 +110,9 @@ static constexpr char* win_cusolver_lib = static constexpr char* win_curand_lib = "curand64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR ".dll;curand64_" CUDA_VERSION_MAJOR ".dll"; +static constexpr char* win_nvjpeg_lib = + "nvjpeg64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR + ".dll;nvjpeg64_" CUDA_VERSION_MAJOR ".dll"; static constexpr char* win_cusolver_lib = "cusolver64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR ".dll;cusolver64_" CUDA_VERSION_MAJOR ".dll"; @@ -330,6 +336,17 @@ void* GetCurandDsoHandle() { #endif } +void* GetNvjpegDsoHandle() { +#if defined(__APPLE__) || defined(__OSX__) + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libnvjpeg.dylib"); +#elif defined(_WIN32) && defined(PADDLE_WITH_CUDA) + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_nvjpeg_lib, true, + {cuda_lib_path}); +#else + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libnvjpeg.so"); +#endif +} + void* GetCusolverDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcusolver.dylib"); diff --git a/paddle/fluid/platform/dynload/dynamic_loader.h b/paddle/fluid/platform/dynload/dynamic_loader.h index 8424160931690..9ab6dca0126bc 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.h +++ b/paddle/fluid/platform/dynload/dynamic_loader.h @@ -29,6 +29,7 @@ void* GetCublasDsoHandle(); void* GetCUDNNDsoHandle(); void* GetCUPTIDsoHandle(); void* GetCurandDsoHandle(); +void* GetNvjpegDsoHandle(); void* GetCusolverDsoHandle(); void* GetNVRTCDsoHandle(); void* GetCUDADsoHandle(); diff --git a/paddle/fluid/platform/dynload/nvjpeg.cc b/paddle/fluid/platform/dynload/nvjpeg.cc new file mode 100644 index 0000000000000..eb0ad78b9b73c --- /dev/null +++ b/paddle/fluid/platform/dynload/nvjpeg.cc @@ -0,0 +1,27 @@ +/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/platform/dynload/nvjpeg.h" + +namespace paddle { +namespace platform { +namespace dynload { + +std::once_flag nvjpeg_dso_flag; +void *nvjpeg_dso_handle; + +#define DEFINE_WRAP(__name) DynLoad__##__name __name + +NVJPEG_RAND_ROUTINE_EACH(DEFINE_WRAP); + +} // namespace dynload +} // namespace platform +} // namespace paddle diff --git a/paddle/fluid/platform/dynload/nvjpeg.h b/paddle/fluid/platform/dynload/nvjpeg.h new file mode 100644 index 0000000000000..ae457b2958f5d --- /dev/null +++ b/paddle/fluid/platform/dynload/nvjpeg.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ +#pragma once + +#ifdef PADDLE_WITH_CUDA +#include +#include // NOLINT + +#include "paddle/fluid/platform/dynload/dynamic_loader.h" +#include "paddle/fluid/platform/port.h" + +namespace paddle { +namespace platform { +namespace dynload { +extern std::once_flag nvjpeg_dso_flag; +extern void *nvjpeg_dso_handle; + +#define DECLARE_DYNAMIC_LOAD_NVJPEG_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + nvjpegStatus_t operator()(Args... args) { \ + using nvjpegFunc = decltype(&::__name); \ + std::call_once(nvjpeg_dso_flag, []() { \ + nvjpeg_dso_handle = paddle::platform::dynload::GetNvjpegDsoHandle(); \ + }); \ + static void *p_##__name = dlsym(nvjpeg_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ + extern DynLoad__##__name __name + +#define NVJPEG_RAND_ROUTINE_EACH(__macro) \ + __macro(nvjpegCreateSimple); \ + __macro(nvjpegJpegStateCreate); \ + __macro(nvjpegGetImageInfo); \ + __macro(nvjpegJpegStateDestroy); \ + __macro(nvjpegDecode); + +NVJPEG_RAND_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_NVJPEG_WRAP); + +} // namespace dynload +} // namespace platform +} // namespace paddle + +#endif diff --git a/python/paddle/tests/test_read_file.py b/python/paddle/tests/test_read_file.py new file mode 100644 index 0000000000000..fbcba9a6bbf7b --- /dev/null +++ b/python/paddle/tests/test_read_file.py @@ -0,0 +1,67 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import cv2 +import shutil +import unittest +import numpy as np + +import paddle +from paddle.vision.ops import read_file, decode_jpeg + + +class TestReadFile(unittest.TestCase): + def setUp(self): + fake_img = (np.random.random((400, 300, 3)) * 255).astype('uint8') + cv2.imwrite('fake.jpg', fake_img) + + def tearDown(self): + os.remove('fake.jpg') + + def read_file_decode_jpeg(self): + if not paddle.is_compiled_with_cuda(): + return + + img_bytes = read_file('fake.jpg') + + img = decode_jpeg(img_bytes, mode='gray') + img = decode_jpeg(img_bytes, mode='rgb') + + img = decode_jpeg(img_bytes) + + img_cv2 = cv2.imread('fake.jpg') + if paddle.in_dynamic_mode(): + np.testing.assert_equal(img.shape, img_cv2.transpose(2, 0, 1).shape) + else: + place = paddle.CUDAPlace(0) + exe = paddle.static.Executor(place) + exe.run(paddle.static.default_startup_program()) + out = exe.run(paddle.static.default_main_program(), + fetch_list=[img]) + + np.testing.assert_equal(out[0].shape, + img_cv2.transpose(2, 0, 1).shape) + + def test_read_file_decode_jpeg_dynamic(self): + self.read_file_decode_jpeg() + + def test_read_file_decode_jpeg_static(self): + paddle.enable_static() + self.read_file_decode_jpeg() + paddle.disable_static() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index 47425476a656a..60a7a90c9be89 100644 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -22,7 +22,10 @@ from paddle.common_ops_import import * -__all__ = ['yolo_loss', 'yolo_box', 'deform_conv2d', 'DeformConv2D'] +__all__ = [ + 'yolo_loss', 'yolo_box', 'deform_conv2d', 'DeformConv2D', 'read_file', + 'decode_jpeg' +] def yolo_loss(x, @@ -782,3 +785,95 @@ def forward(self, x, offset, mask=None): groups=self._groups, mask=mask) return out + + +def read_file(filename, name=None): + """ + Reads and outputs the bytes contents of a file as a uint8 Tensor + with one dimension. + + Args: + filename (str): Path of the file to be read. + name (str, optional): The default value is None. Normally there is no + need for user to set this property. For more information, please + refer to :ref:`api_guide_Name`. + + Returns: + A uint8 tensor. + + Examples: + .. code-block:: python + + import cv2 + import paddle + + fake_img = (np.random.random( + (400, 300, 3)) * 255).astype('uint8') + + cv2.imwrite('fake.jpg', fake_img) + + img_bytes = paddle.vision.ops.read_file('fake.jpg') + + print(img_bytes.shape) + + """ + + if in_dygraph_mode(): + return core.ops.read_file('filename', filename) + + inputs = dict() + attrs = {'filename': filename} + + helper = LayerHelper("read_file", **locals()) + out = helper.create_variable_for_type_inference('uint8') + helper.append_op( + type="read_file", inputs=inputs, attrs=attrs, outputs={"Out": out}) + + return out + + +def decode_jpeg(x, mode='unchanged', name=None): + """ + Decodes a JPEG image into a 3 dimensional RGB Tensor or 1 dimensional Gray Tensor. + Optionally converts the image to the desired format. + The values of the output tensor are uint8 between 0 and 255. + + Args: + x (Tensor): A one dimensional uint8 tensor containing the raw bytes + of the JPEG image. + mode (str): The read mode used for optionally converting the image. + Default: 'unchanged'. + name (str, optional): The default value is None. Normally there is no + need for user to set this property. For more information, please + refer to :ref:`api_guide_Name`. + Returns: + Tensor: A decoded image tensor with shape (imge_channels, image_height, image_width) + + Examples: + .. code-block:: python + import cv2 + import paddle + + fake_img = (np.random.random( + (400, 300, 3)) * 255).astype('uint8') + + cv2.imwrite('fake.jpg', fake_img) + + img_bytes = paddle.vision.ops.read_file('fake.jpg') + img = paddle.vision.ops.decode_jpeg(img_bytes) + + print(img.shape) + """ + + if in_dygraph_mode(): + return core.ops.decode_jpeg(x, "mode", mode) + + inputs = {'X': x} + attrs = {"mode": mode} + + helper = LayerHelper("decode_jpeg", **locals()) + out = helper.create_variable_for_type_inference('uint8') + helper.append_op( + type="decode_jpeg", inputs=inputs, attrs=attrs, outputs={"Out": out}) + + return out From 1a417a4c74364ec5d1ce5bbd411fee0d2c76041b Mon Sep 17 00:00:00 2001 From: ceci3 Date: Fri, 30 Apr 2021 12:58:07 +0800 Subject: [PATCH 021/156] remove is_test=True in grad (#32683) --- paddle/fluid/operators/batch_norm_op.cc | 11 +++-------- paddle/fluid/operators/batch_norm_op.cu | 9 ++------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/operators/batch_norm_op.cc b/paddle/fluid/operators/batch_norm_op.cc index fc31885824b55..edad20435b41c 100644 --- a/paddle/fluid/operators/batch_norm_op.cc +++ b/paddle/fluid/operators/batch_norm_op.cc @@ -575,7 +575,7 @@ class BatchNormGradKernel // SavedVariance have been reverted in forward operator const auto *saved_inv_variance = ctx.Input("SavedVariance"); const std::string data_layout_str = ctx.Attr("data_layout"); - const bool use_global_stats = ctx.Attr("use_global_stats"); + bool use_global_stats = ctx.Attr("use_global_stats"); const bool is_test = ctx.Attr("is_test"); const float epsilon = ctx.Attr("epsilon"); const DataLayout data_layout = @@ -585,6 +585,8 @@ class BatchNormGradKernel auto *d_scale = ctx.Output(framework::GradVarName("Scale")); auto *d_bias = ctx.Output(framework::GradVarName("Bias")); + use_global_stats = is_test || use_global_stats; + // batch_norm with inplace as false will take X as grad input, which // is same as cuDNN batch_norm backward calculation, batch_norm // with inplace as true only take Y as input and X should be calculate @@ -605,13 +607,6 @@ class BatchNormGradKernel "X@GRAD and Y@GRAD inplaced in non-inplace mode")); } - PADDLE_ENFORCE_EQ( - is_test, false, - platform::errors::InvalidArgument( - "`is_test = True` CANNOT be used in train program. If " - "you want to use global status in pre_train model, " - "please set `use_global_stats = True`")); - // Get the size for each dimension. // NCHW [batch_size, in_channels, in_height, in_width] const auto &x_dims = x->dims(); diff --git a/paddle/fluid/operators/batch_norm_op.cu b/paddle/fluid/operators/batch_norm_op.cu index 41dc87ac1ba47..6fc78732b1063 100644 --- a/paddle/fluid/operators/batch_norm_op.cu +++ b/paddle/fluid/operators/batch_norm_op.cu @@ -817,7 +817,7 @@ class BatchNormGradKernel platform::errors::InvalidArgument("It must use CUDAPlace.")); double epsilon = static_cast(ctx.Attr("epsilon")); const std::string data_layout_str = ctx.Attr("data_layout"); - const bool use_global_stats = ctx.Attr("use_global_stats"); + bool use_global_stats = ctx.Attr("use_global_stats"); const DataLayout data_layout = framework::StringToDataLayout(data_layout_str); @@ -850,12 +850,7 @@ class BatchNormGradKernel } const bool is_test = ctx.Attr("is_test"); - PADDLE_ENFORCE_EQ( - is_test, false, - platform::errors::InvalidArgument( - "`is_test = True` CANNOT be used in train program. If " - "you want to use global status in pre_train model, " - "please set `use_global_stats = True`")); + use_global_stats = is_test || use_global_stats; const auto &x_dims = x->dims(); From 097d5f52ba7aa1477a01284abb8356a8331d6172 Mon Sep 17 00:00:00 2001 From: pangyoki Date: Fri, 30 Apr 2021 16:31:28 +0800 Subject: [PATCH 022/156] Add 12 inplace APIs including auto generated (#32573) (#32699) * add relu6_ hardsigmoid_ leaky_relu_ Inplace APIs * add softmax_with_cross_entropy_ Inplace API * add clip_ scale_ add_ subtract_ Inplace APIs * add wlist * fix parameter of scale api * add add_n_ Inplace API and remove log_ Inplace API * fix elementwise_add_ and elementwise_sub_ broadcast problem * elementwise inplace api give error message before run the op * use broadcast_shape in elementwise inplace op * add 8 inplace apis that is auto generated * add unittest for all inplace apis * add decorator for inplace apis in static mode * fix windows blas fail of exp inplace api, change array_equal to allclose * add flatten inplace api * add flatten unittest * fix flatten unittest * add decorator * fix grad.numpy in test_pylayer_op * unsupport softmax_with_cross_entropy_ * add test_inplace_softmax_with_cross_entropy to static_mode_white_list * delete __all__ in inplace_utils * delete activation inplace function and add Tensor.inplace_func * change paddle.inplace_ to Tensor.inplace_ * fix little problem * add paddle in inplace_utils --- paddle/fluid/imperative/basic_engine.cc | 3 +- paddle/fluid/operators/flatten_op.h | 37 +-- python/paddle/fluid/dygraph/__init__.py | 2 + python/paddle/fluid/dygraph/inplace_utils.py | 38 +++ .../fluid/layers/layer_function_generator.py | 32 +- python/paddle/fluid/layers/ops.py | 21 +- .../fluid/tests/unittests/test_clip_op.py | 48 +-- .../unittests/test_elementwise_add_op.py | 74 ++++- .../unittests/test_elementwise_sub_op.py | 106 +++++++ .../test_flatten_contiguous_range_op.py | 42 +++ .../fluid/tests/unittests/test_inplace.py | 117 +++++++- .../test_inplace_auto_generated_apis.py | 281 ++++++++++++++++++ .../fluid/tests/unittests/test_scale_op.py | 42 +++ python/paddle/nn/functional/activation.py | 27 +- python/paddle/tensor/__init__.py | 24 ++ python/paddle/tensor/manipulation.py | 108 ++++--- python/paddle/tensor/math.py | 82 ++++- tools/wlist.json | 48 +++ 18 files changed, 997 insertions(+), 135 deletions(-) create mode 100644 python/paddle/fluid/dygraph/inplace_utils.py create mode 100644 python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py diff --git a/paddle/fluid/imperative/basic_engine.cc b/paddle/fluid/imperative/basic_engine.cc index d5350744e4c55..896918a607106 100644 --- a/paddle/fluid/imperative/basic_engine.cc +++ b/paddle/fluid/imperative/basic_engine.cc @@ -408,7 +408,8 @@ void BasicEngine::Execute() { VLOG(10) << "create temporary var of " << var->Name() << " for sum gradient within this graph!"; } else if (!inplace_grad_name_map.empty() && - inplace_grad_name_map.count(pair.first)) { + inplace_grad_name_map.count(pair.first) && + bwd_ins.count(inplace_grad_name_map.at(pair.first))) { // When calculate Inplace grad op, create a new output var. // If a tmp var has been created, there is no need to create it // again. diff --git a/paddle/fluid/operators/flatten_op.h b/paddle/fluid/operators/flatten_op.h index 1b2f1db1b07cd..efcb0cbe2e2a8 100644 --- a/paddle/fluid/operators/flatten_op.h +++ b/paddle/fluid/operators/flatten_op.h @@ -120,23 +120,9 @@ template class FlattenContiguousRangeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { - auto &start_axis = context.Attr("start_axis"); - auto &stop_axis = context.Attr("stop_axis"); - auto *in = context.Input("X"); - auto x_dims = in->dims(); - int in_dims_size = x_dims.size(); - int real_start_axis = start_axis, real_stop_axis = stop_axis; - if (start_axis < 0) { - real_start_axis = start_axis + in_dims_size; - } - if (stop_axis < 0) { - real_stop_axis = stop_axis + in_dims_size; - } auto *out = context.Output("Out"); - - auto out_dims = framework::make_ddim( - GetOutputShape(real_start_axis, real_stop_axis, x_dims)); + auto out_dims = out->dims(); out->mutable_data(context.GetPlace(), in->type()); framework::TensorCopy( @@ -144,27 +130,6 @@ class FlattenContiguousRangeKernel : public framework::OpKernel { context.template device_context(), out); out->Resize(out_dims); } - static std::vector GetOutputShape(const int start_axis, - const int stop_axis, - const framework::DDim &in_dims) { - int64_t outer = 1; - std::vector out_shape; - int in_dims_size = in_dims.size(); - out_shape.reserve(in_dims_size - stop_axis + start_axis); - - for (int i = 0; i < start_axis; ++i) { - out_shape.push_back(in_dims[i]); - } - for (int i = start_axis; i <= stop_axis; i++) { - outer *= in_dims[i]; - } - out_shape.push_back(outer); - for (int i = stop_axis + 1; i < in_dims_size; i++) { - out_shape.push_back(in_dims[i]); - } - - return out_shape; - } }; template diff --git a/python/paddle/fluid/dygraph/__init__.py b/python/paddle/fluid/dygraph/__init__.py index cf270ced3b704..d66e33097833a 100644 --- a/python/paddle/fluid/dygraph/__init__.py +++ b/python/paddle/fluid/dygraph/__init__.py @@ -58,6 +58,8 @@ from .math_op_patch import monkey_patch_math_varbase +from .inplace_utils import inplace_apis_in_dygraph_only + __all__ = [] __all__ += layers.__all__ __all__ += base.__all__ diff --git a/python/paddle/fluid/dygraph/inplace_utils.py b/python/paddle/fluid/dygraph/inplace_utils.py new file mode 100644 index 0000000000000..c1f7ef9b691c0 --- /dev/null +++ b/python/paddle/fluid/dygraph/inplace_utils.py @@ -0,0 +1,38 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..wrapped_decorator import wrap_decorator +from ..framework import in_dygraph_mode +import warnings +import paddle + + +# NOTE(pangyoki): The Inplace APIs with underline(`_`) is only valid for the method of calling `core.ops` +# in dygraph mode. If static mode is used, the inplace mechanism will not be used, and the static method +# of the original API will be called. +def _inplace_apis_in_dygraph_only_(func): + def __impl__(*args, **kwargs): + if not in_dygraph_mode(): + origin_api_name = func.__name__[:-1] + warnings.warn( + "In static mode, {}() is the same as {}() and does not perform inplace operation.". + format(func.__name__, origin_api_name)) + origin_func = "{}.{}".format(func.__module__, origin_api_name) + return eval(origin_func)(*args, **kwargs) + return func(*args, **kwargs) + + return __impl__ + + +inplace_apis_in_dygraph_only = wrap_decorator(_inplace_apis_in_dygraph_only_) diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index 708692c215fb0..6e52ea04a195a 100755 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -25,7 +25,8 @@ from ..data_feeder import check_variable_and_dtype __all__ = [ - 'generate_layer_fn', 'generate_activation_fn', 'autodoc', 'templatedoc' + 'generate_layer_fn', 'generate_activation_fn', 'generate_inplace_fn', + 'autodoc', 'templatedoc' ] @@ -283,6 +284,35 @@ def func(x, name=None): return func +def generate_inplace_fn(inplace_op_type): + """Register the Python layer for an Inplace Operator without Attribute. + + Args: + inplace_op_type: The name of the inplace operator to be created. + + This function takes in the inplace operator type (exp_ , ceil_ etc) and + creates the operator functionality. + """ + origin_op_type = inplace_op_type[:-1] + + def func(x, name=None): + if in_dygraph_mode(): + op = getattr(core.ops, inplace_op_type) + return op(x) + warnings.warn( + "In static mode, {}() is the same as {}() and does not perform inplace operation.". + format(inplace_op_type, origin_op_type)) + return generate_activation_fn(origin_op_type)(x, name) + + func.__name__ = inplace_op_type + func.__doc__ = """ +Inplace version of ``{0}`` API, the output Tensor will be inplaced with input ``x``. +Please refer to :ref:`api_fluid_layers_{1}`. +""".format(origin_op_type, origin_op_type) + + return func + + def autodoc(comment=""): def __impl__(func): func.__doc__ = _generate_doc_string_(OpProtoHolder.instance( diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 67cdc6dce5a82..813f671e02070 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -14,7 +14,7 @@ from __future__ import print_function import os -from .layer_function_generator import generate_layer_fn, generate_activation_fn, add_sample_code +from .layer_function_generator import generate_layer_fn, generate_activation_fn, generate_inplace_fn, add_sample_code from .. import core from ..framework import convert_np_dtype_to_dtype_, Variable from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype @@ -55,6 +55,16 @@ 'square', ] +__inplace_unary_func__ = [ + 'exp_', + 'sqrt_', + 'rsqrt_', + 'ceil_', + 'floor_', + 'round_', + 'reciprocal_', +] + __all__ = [] for _OP in set(__all__): @@ -69,6 +79,7 @@ __all__ += __activations_noattr__ __all__ += __unary_func__ +__all__ += __inplace_unary_func__ for _OP in set(__activations_noattr__): _new_OP = _OP @@ -87,6 +98,14 @@ func = deprecated(since="2.0.0", update_to="paddle.%s" % (_new_OP))(func) globals()[_OP] = func +for _OP in set(__inplace_unary_func__): + _new_OP = _OP + if _OP in __deprecated_func_name__: + _new_OP = __deprecated_func_name__[_OP] + func = generate_inplace_fn(_OP) + func = deprecated(since="2.0.0", update_to="paddle.%s" % (_new_OP))(func) + globals()[_OP] = func + add_sample_code(globals()["sigmoid"], r""" Examples: .. code-block:: python diff --git a/python/paddle/fluid/tests/unittests/test_clip_op.py b/python/paddle/fluid/tests/unittests/test_clip_op.py index b05100fc7b433..1833c473d18a9 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_op.py @@ -124,6 +124,9 @@ def test_dtype(): class TestClipAPI(unittest.TestCase): + def _executed_api(self, x, min=None, max=None): + return paddle.clip(x, min, max) + def test_clip(self): paddle.enable_static() data_shape = [1, 9, 9, 4] @@ -136,18 +139,20 @@ def test_clip(self): ) else fluid.CPUPlace() exe = fluid.Executor(place) - out_1 = paddle.clip(images, min=min, max=max) - out_2 = paddle.clip(images, min=0.2, max=0.9) - out_3 = paddle.clip(images, min=0.3) - out_4 = paddle.clip(images, max=0.7) - out_5 = paddle.clip(images, min=min) - out_6 = paddle.clip(images, max=max) - out_7 = paddle.clip(images, max=-1.) - out_8 = paddle.clip(images) - out_9 = paddle.clip(paddle.cast(images, 'float64'), min=0.2, max=0.9) - - out_10 = paddle.clip(paddle.cast(images * 10, 'int32'), min=2, max=8) - out_11 = paddle.clip(paddle.cast(images * 10, 'int64'), min=2, max=8) + out_1 = self._executed_api(images, min=min, max=max) + out_2 = self._executed_api(images, min=0.2, max=0.9) + out_3 = self._executed_api(images, min=0.3) + out_4 = self._executed_api(images, max=0.7) + out_5 = self._executed_api(images, min=min) + out_6 = self._executed_api(images, max=max) + out_7 = self._executed_api(images, max=-1.) + out_8 = self._executed_api(images) + out_9 = self._executed_api( + paddle.cast(images, 'float64'), min=0.2, max=0.9) + out_10 = self._executed_api( + paddle.cast(images * 10, 'int32'), min=2, max=8) + out_11 = self._executed_api( + paddle.cast(images * 10, 'int64'), min=2, max=8) res1, res2, res3, res4, res5, res6, res7, res8, res9, res10, res11 = exe.run( fluid.default_main_program(), @@ -188,12 +193,16 @@ def test_clip_dygraph(self): v_min = paddle.to_tensor(np.array([0.2], dtype=np.float32)) v_max = paddle.to_tensor(np.array([0.8], dtype=np.float32)) - out_1 = paddle.clip(images, min=0.2, max=0.8) - out_2 = paddle.clip(images, min=0.2, max=0.9) - out_3 = paddle.clip(images, min=v_min, max=v_max) + out_1 = self._executed_api(images, min=0.2, max=0.8) + images = paddle.to_tensor(data, dtype='float32') + out_2 = self._executed_api(images, min=0.2, max=0.9) + images = paddle.to_tensor(data, dtype='float32') + out_3 = self._executed_api(images, min=v_min, max=v_max) - out_4 = paddle.clip(paddle.cast(images * 10, 'int32'), min=2, max=8) - out_5 = paddle.clip(paddle.cast(images * 10, 'int64'), min=2, max=8) + out_4 = self._executed_api( + paddle.cast(images * 10, 'int32'), min=2, max=8) + out_5 = self._executed_api( + paddle.cast(images * 10, 'int64'), min=2, max=8) self.assertTrue(np.allclose(out_1.numpy(), data.clip(0.2, 0.8))) self.assertTrue(np.allclose(out_2.numpy(), data.clip(0.2, 0.9))) @@ -212,5 +221,10 @@ def test_errors(self): paddle.disable_static() +class TestInplaceClipAPI(TestClipAPI): + def _executed_api(self, x, min=None, max=None): + return x.clip_(min, max) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py index cc362005f3311..f24d41d4d00f9 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py @@ -408,13 +408,16 @@ def test_errors(self): self.assertRaises(TypeError, fluid.layers.elementwise_add, x2, y2) -class TestAddOp(unittest.TestCase): +class TestAddApi(unittest.TestCase): + def _executed_api(self, x, y, name=None): + return paddle.add(x, y, name) + def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="float32") y = fluid.data(name='y', shape=[2, 3], dtype='float32') - y_1 = paddle.add(x, y, name='add_res') + y_1 = self._executed_api(x, y, name='add_res') self.assertEqual(('add_res' in y_1.name), True) def test_declarative(self): @@ -428,7 +431,7 @@ def gen_data(): x = fluid.data(name="x", shape=[3], dtype='float32') y = fluid.data(name="y", shape=[3], dtype='float32') - z = paddle.add(x, y) + z = self._executed_api(x, y) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -442,12 +445,75 @@ def test_dygraph(self): np_y = np.array([1, 5, 2]).astype('float64') x = fluid.dygraph.to_variable(np_x) y = fluid.dygraph.to_variable(np_y) - z = paddle.add(x, y) + z = self._executed_api(x, y) np_z = z.numpy() z_expected = np.array([3., 8., 6.]) self.assertEqual((np_z == z_expected).all(), True) +class TestAddInplaceApi(TestAddApi): + def _executed_api(self, x, y, name=None): + return x.add_(y, name) + + +class TestAddInplaceBroadcastSuccess(unittest.TestCase): + def init_data(self): + self.x_numpy = np.random.rand(2, 3, 4).astype('float') + self.y_numpy = np.random.rand(3, 4).astype('float') + + def test_broadcast_success(self): + paddle.disable_static() + self.init_data() + x = paddle.to_tensor(self.x_numpy) + y = paddle.to_tensor(self.y_numpy) + inplace_result = x.add_(y) + numpy_result = self.x_numpy + self.y_numpy + self.assertEqual((inplace_result.numpy() == numpy_result).all(), True) + paddle.enable_static() + + +class TestAddInplaceBroadcastSuccess2(TestAddInplaceBroadcastSuccess): + def init_data(self): + self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') + self.y_numpy = np.random.rand(3, 1).astype('float') + + +class TestAddInplaceBroadcastSuccess3(TestAddInplaceBroadcastSuccess): + def init_data(self): + self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') + self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') + + +class TestAddInplaceBroadcastError(unittest.TestCase): + def init_data(self): + self.x_numpy = np.random.rand(3, 4).astype('float') + self.y_numpy = np.random.rand(2, 3, 4).astype('float') + + def test_broadcast_errors(self): + paddle.disable_static() + self.init_data() + x = paddle.to_tensor(self.x_numpy) + y = paddle.to_tensor(self.y_numpy) + + def broadcast_shape_error(): + x.add_(y) + + self.assertRaises(ValueError, broadcast_shape_error) + paddle.enable_static() + + +class TestAddInplaceBroadcastError2(TestAddInplaceBroadcastError): + def init_data(self): + self.x_numpy = np.random.rand(2, 1, 4).astype('float') + self.y_numpy = np.random.rand(2, 3, 4).astype('float') + + +class TestAddInplaceBroadcastError3(TestAddInplaceBroadcastError): + def init_data(self): + self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') + self.y_numpy = np.random.rand(2, 3, 4).astype('float') + + class TestComplexElementwiseAddOp(OpTest): def setUp(self): self.op_type = "elementwise_add" diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py index c5372d5b758a8..2594c96eebd69 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py @@ -16,6 +16,7 @@ import unittest import numpy as np import paddle +import paddle.fluid as fluid from op_test import OpTest, skip_check_grad_ci @@ -237,6 +238,111 @@ def init_grad_input_output(self): self.grad_y = -self.grad_out +class TestSubtractApi(unittest.TestCase): + def _executed_api(self, x, y, name=None): + return paddle.subtract(x, y, name) + + def test_name(self): + with fluid.program_guard(fluid.Program()): + x = fluid.data(name="x", shape=[2, 3], dtype="float32") + y = fluid.data(name='y', shape=[2, 3], dtype='float32') + + y_1 = self._executed_api(x, y, name='subtract_res') + self.assertEqual(('subtract_res' in y_1.name), True) + + def test_declarative(self): + with fluid.program_guard(fluid.Program()): + + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = self._executed_api(x, y) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) + z_expected = np.array([1., -2., 2.]) + self.assertEqual((z_value == z_expected).all(), True) + + def test_dygraph(self): + with fluid.dygraph.guard(): + np_x = np.array([2, 3, 4]).astype('float64') + np_y = np.array([1, 5, 2]).astype('float64') + x = fluid.dygraph.to_variable(np_x) + y = fluid.dygraph.to_variable(np_y) + z = self._executed_api(x, y) + np_z = z.numpy() + z_expected = np.array([1., -2., 2.]) + self.assertEqual((np_z == z_expected).all(), True) + + +class TestSubtractInplaceApi(TestSubtractApi): + def _executed_api(self, x, y, name=None): + return x.subtract_(y, name) + + +class TestSubtractInplaceBroadcastSuccess(unittest.TestCase): + def init_data(self): + self.x_numpy = np.random.rand(2, 3, 4).astype('float') + self.y_numpy = np.random.rand(3, 4).astype('float') + + def test_broadcast_success(self): + paddle.disable_static() + self.init_data() + x = paddle.to_tensor(self.x_numpy) + y = paddle.to_tensor(self.y_numpy) + inplace_result = x.subtract_(y) + numpy_result = self.x_numpy - self.y_numpy + self.assertEqual((inplace_result.numpy() == numpy_result).all(), True) + paddle.enable_static() + + +class TestSubtractInplaceBroadcastSuccess2(TestSubtractInplaceBroadcastSuccess): + def init_data(self): + self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') + self.y_numpy = np.random.rand(3, 1).astype('float') + + +class TestSubtractInplaceBroadcastSuccess3(TestSubtractInplaceBroadcastSuccess): + def init_data(self): + self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') + self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') + + +class TestSubtractInplaceBroadcastError(unittest.TestCase): + def init_data(self): + self.x_numpy = np.random.rand(3, 4).astype('float') + self.y_numpy = np.random.rand(2, 3, 4).astype('float') + + def test_broadcast_errors(self): + paddle.disable_static() + self.init_data() + x = paddle.to_tensor(self.x_numpy) + y = paddle.to_tensor(self.y_numpy) + + def broadcast_shape_error(): + x.subtract_(y) + + self.assertRaises(ValueError, broadcast_shape_error) + paddle.enable_static() + + +class TestSubtractInplaceBroadcastError2(TestSubtractInplaceBroadcastError): + def init_data(self): + self.x_numpy = np.random.rand(2, 1, 4).astype('float') + self.y_numpy = np.random.rand(2, 3, 4).astype('float') + + +class TestSubtractInplaceBroadcastError3(TestSubtractInplaceBroadcastError): + def init_data(self): + self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') + self.y_numpy = np.random.rand(2, 3, 4).astype('float') + + if __name__ == '__main__': paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py b/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py index d6cc6ecffc106..bc9ff3697717d 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py @@ -182,6 +182,30 @@ def test_InputError(): self.assertRaises(ValueError, test_InputError) +class TestStaticFlattenPythonAPI(unittest.TestCase): + def execute_api(self, x, start_axis=0, stop_axis=-1): + return paddle.flatten(x, start_axis, stop_axis) + + def test_static_api(self): + paddle.enable_static() + np_x = np.random.rand(2, 3, 4, 4).astype('float32') + + main_prog = paddle.static.Program() + with paddle.static.program_guard(main_prog, paddle.static.Program()): + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32') + out = self.execute_api(x, start_axis=-2, stop_axis=-1) + + exe = paddle.static.Executor(place=paddle.CPUPlace()) + fetch_out = exe.run(main_prog, feed={"x": np_x}, fetch_list=[out]) + self.assertTrue((2, 3, 16) == fetch_out[0].shape) + + +class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): + def execute_api(self, x, start_axis=0, stop_axis=-1): + return x.flatten_(start_axis, stop_axis) + + class TestFlattenPython(unittest.TestCase): def test_python_api(self): image_shape = (2, 3, 4, 4) @@ -204,5 +228,23 @@ def test_Negative(): self.assertTrue((2, 3, 16) == res_shape) +class TestDygraphInplaceFlattenPython(unittest.TestCase): + def test_python_api(self): + image_shape = (2, 3, 4, 4) + x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * + image_shape[3]).reshape(image_shape) / 100. + x = x.astype('float32') + + def test_Negative(): + paddle.disable_static() + img = paddle.to_tensor(x) + out = img.flatten_(start_axis=-2, stop_axis=-1) + return out.numpy().shape + + res_shape = test_Negative() + self.assertTrue((2, 3, 16) == res_shape) + paddle.enable_static() + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_inplace.py b/python/paddle/fluid/tests/unittests/test_inplace.py index 7b9becacd82c1..3d158763527e7 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace.py +++ b/python/paddle/fluid/tests/unittests/test_inplace.py @@ -98,11 +98,15 @@ def test_backward_success_2(self): class TestDygraphInplace(unittest.TestCase): def setUp(self): self.init_data() + self.set_np_compare_func() def init_data(self): - self.input_var_numpy = np.random.rand(2, 3, 1) + self.input_var_numpy = np.random.uniform(-5, 5, [10, 20, 1]) self.dtype = "float32" + def set_np_compare_func(self): + self.np_compare = np.array_equal + def non_inplace_api_processing(self, var): return paddle.squeeze(var) @@ -190,7 +194,7 @@ def test_backward_success_1(self): loss.backward() grad_var_a = var_a.grad.numpy() - self.assertTrue(np.array_equal(grad_var_a_inplace, grad_var_a)) + self.assertTrue(self.np_compare(grad_var_a_inplace, grad_var_a)) def test_backward_success_2(self): # Although var_b is modified inplace after using it, it does not used in gradient computation. @@ -244,6 +248,14 @@ def inplace_api_processing(self, var): return paddle.reshape_(var, [-1]) +class TestDygraphInplaceFlatten(TestDygraphInplace): + def non_inplace_api_processing(self, var): + return var.flatten() + + def inplace_api_processing(self, var): + return var.flatten_() + + class TestDygraphInplaceScatter(TestDygraphInplace): def init_data(self): self.input_var_numpy = np.array([[1, 1], [2, 2], [3, 3]]) @@ -296,5 +308,106 @@ def inplace_api_processing(self, var): return paddle.tanh_(var) +class TestDygraphInplaceCeil(TestDygraphInplace): + def non_inplace_api_processing(self, var): + return var.ceil() + + def inplace_api_processing(self, var): + return var.ceil_() + + +class TestDygraphInplaceFloor(TestDygraphInplace): + def non_inplace_api_processing(self, var): + return var.floor() + + def inplace_api_processing(self, var): + return var.floor_() + + +class TestDygraphInplaceExp(TestDygraphInplace): + def set_np_compare_func(self): + self.np_compare = np.allclose + + def non_inplace_api_processing(self, var): + return var.exp() + + def inplace_api_processing(self, var): + return var.exp_() + + +class TestDygraphInplaceReciprocal(TestDygraphInplace): + def non_inplace_api_processing(self, var): + return var.reciprocal() + + def inplace_api_processing(self, var): + return var.reciprocal_() + + +class TestDygraphInplaceRound(TestDygraphInplace): + def non_inplace_api_processing(self, var): + return var.round() + + def inplace_api_processing(self, var): + return var.round_() + + +class TestDygraphInplaceSqrt(TestDygraphInplace): + def init_data(self): + self.input_var_numpy = np.random.uniform(0, 5, [10, 20, 1]) + self.dtype = "float32" + + def non_inplace_api_processing(self, var): + return var.sqrt() + + def inplace_api_processing(self, var): + return var.sqrt_() + + +class TestDygraphInplaceRsqrt(TestDygraphInplaceSqrt): + def non_inplace_api_processing(self, var): + return var.rsqrt() + + def inplace_api_processing(self, var): + return var.rsqrt_() + + +class TestDygraphInplaceClip(TestDygraphInplace): + def non_inplace_api_processing(self, var): + return var.clip(0.6, 1.5) + + def inplace_api_processing(self, var): + return var.clip_(0.6, 1.5) + + +class TestDygraphInplaceScale(TestDygraphInplace): + def non_inplace_api_processing(self, var): + return var.scale(scale=2.0, bias=3.0) + + def inplace_api_processing(self, var): + return var.scale_(scale=2.0, bias=3.0) + + +class TestDygraphInplaceAdd(TestDygraphInplace): + def init_data(self): + self.input_var_numpy = np.random.rand(2, 3, 4) + self.dtype = "float32" + input_var_numpy_2 = np.random.rand(2, 3, 4).astype(self.dtype) + self.input_var_2 = paddle.to_tensor(input_var_numpy_2) + + def non_inplace_api_processing(self, var): + return var.add(self.input_var_2) + + def inplace_api_processing(self, var): + return var.add_(self.input_var_2) + + +class TestDygraphInplaceSubtract(TestDygraphInplaceAdd): + def non_inplace_api_processing(self, var): + return var.subtract(self.input_var_2) + + def inplace_api_processing(self, var): + return var.subtract_(self.input_var_2) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py b/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py new file mode 100644 index 0000000000000..abc8849b614f7 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py @@ -0,0 +1,281 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np + +from op_test import OpTest +import paddle +import paddle.fluid as fluid +from paddle.static import Program, program_guard + + +# In static mode, inplace strategy will not be used in Inplace APIs. +class TestStaticAutoGeneratedAPI(unittest.TestCase): + def setUp(self): + paddle.enable_static() + self.init_data() + self.set_np_compare_func() + + def init_data(self): + self.dtype = 'float32' + self.shape = [10, 20] + self.np_x = np.random.uniform(-5, 5, self.shape).astype(self.dtype) + + def set_np_compare_func(self): + self.np_compare = np.array_equal + + def executed_paddle_api(self, x): + return x.ceil() + + def executed_numpy_api(self, x): + return np.ceil(x) + + def test_api(self): + main_prog = Program() + with program_guard(main_prog, Program()): + x = paddle.static.data(name="x", shape=self.shape, dtype=self.dtype) + out = self.executed_paddle_api(x) + + exe = paddle.static.Executor(place=paddle.CPUPlace()) + fetch_x, fetch_out = exe.run(main_prog, + feed={"x": self.np_x}, + fetch_list=[x, out]) + + self.assertTrue(np.array_equal(fetch_x, self.np_x)) + self.assertTrue( + self.np_compare(fetch_out, self.executed_numpy_api(self.np_x))) + + +class TestStaticInplaceAutoGeneratedAPI(TestStaticAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.ceil_() + + +class TestStaticFloorAPI(TestStaticAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.floor() + + def executed_numpy_api(self, x): + return np.floor(x) + + +class TestStaticInplaceFloorAPI(TestStaticFloorAPI): + def executed_paddle_api(self, x): + return x.floor_() + + +class TestStaticExpAPI(TestStaticAutoGeneratedAPI): + def set_np_compare_func(self): + self.np_compare = np.allclose + + def executed_paddle_api(self, x): + return x.exp() + + def executed_numpy_api(self, x): + return np.exp(x) + + +class TestStaticInplaceExpAPI(TestStaticExpAPI): + def executed_paddle_api(self, x): + return x.exp_() + + +class TestStaticReciprocalAPI(TestStaticAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.reciprocal() + + def executed_numpy_api(self, x): + return np.reciprocal(x) + + +class TestStaticInplaceReciprocalAPI(TestStaticReciprocalAPI): + def executed_paddle_api(self, x): + return x.reciprocal_() + + +class TestStaticRoundAPI(TestStaticAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.round() + + def executed_numpy_api(self, x): + return np.round(x) + + +class TestStaticInplaceRoundAPI(TestStaticRoundAPI): + def executed_paddle_api(self, x): + return x.round_() + + +class TestStaticSqrtAPI(TestStaticAutoGeneratedAPI): + def init_data(self): + self.dtype = 'float32' + self.shape = [10, 20] + self.np_x = np.random.uniform(0, 5, self.shape).astype(self.dtype) + + def set_np_compare_func(self): + self.np_compare = np.allclose + + def executed_paddle_api(self, x): + return x.sqrt() + + def executed_numpy_api(self, x): + return np.sqrt(x) + + +class TestStaticInplaceSqrtAPI(TestStaticSqrtAPI): + def executed_paddle_api(self, x): + return x.sqrt_() + + +class TestStaticRsqrtAPI(TestStaticSqrtAPI): + def executed_paddle_api(self, x): + return x.rsqrt() + + def executed_numpy_api(self, x): + return 1 / np.sqrt(x) + + +class TestStaticInplaceRsqrtAPI(TestStaticRsqrtAPI): + def executed_paddle_api(self, x): + return x.rsqrt_() + + +# In dygraph mode, inplace strategy will be used in Inplace APIs. +class TestDygraphAutoGeneratedAPI(unittest.TestCase): + def setUp(self): + paddle.disable_static() + self.init_data() + self.set_np_compare_func() + + def init_data(self): + self.dtype = 'float32' + self.shape = [10, 20] + self.np_x = np.random.uniform(-5, 5, self.shape).astype(self.dtype) + + def set_np_compare_func(self): + self.np_compare = np.array_equal + + def executed_paddle_api(self, x): + return x.ceil() + + def executed_numpy_api(self, x): + return np.ceil(x) + + def test_api(self): + x = paddle.to_tensor(self.np_x, dtype=self.dtype) + out = self.executed_paddle_api(x) + + self.assertTrue( + self.np_compare(out.numpy(), self.executed_numpy_api(self.np_x))) + + +class TestDygraphInplaceAutoGeneratedAPI(TestDygraphAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.ceil_() + + +class TestDygraphFloorAPI(TestDygraphAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.floor() + + def executed_numpy_api(self, x): + return np.floor(x) + + +class TestDygraphInplaceFloorAPI(TestDygraphFloorAPI): + def executed_paddle_api(self, x): + return x.floor_() + + +class TestDygraphExpAPI(TestDygraphAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.exp() + + def executed_numpy_api(self, x): + return np.exp(x) + + def set_np_compare_func(self): + self.np_compare = np.allclose + + +class TestDygraphInplaceExpAPI(TestDygraphExpAPI): + def executed_paddle_api(self, x): + return x.exp_() + + +class TestDygraphReciprocalAPI(TestDygraphAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.reciprocal() + + def executed_numpy_api(self, x): + return np.reciprocal(x) + + +class TestDygraphInplaceReciprocalAPI(TestDygraphReciprocalAPI): + def executed_paddle_api(self, x): + return x.reciprocal_() + + +class TestDygraphRoundAPI(TestDygraphAutoGeneratedAPI): + def executed_paddle_api(self, x): + return x.round() + + def executed_numpy_api(self, x): + return np.round(x) + + +class TestDygraphInplaceRoundAPI(TestDygraphRoundAPI): + def executed_paddle_api(self, x): + return x.round_() + + +class TestDygraphSqrtAPI(TestDygraphAutoGeneratedAPI): + def init_data(self): + self.dtype = 'float32' + self.shape = [10, 20] + self.np_x = np.random.uniform(0, 100, self.shape).astype(self.dtype) + + def set_np_compare_func(self): + self.np_compare = np.allclose + + def executed_paddle_api(self, x): + return x.sqrt() + + def executed_numpy_api(self, x): + return np.sqrt(x) + + +class TestDygraphInplaceSqrtAPI(TestDygraphSqrtAPI): + def executed_paddle_api(self, x): + return x.sqrt_() + + +class TestDygraphRsqrtAPI(TestDygraphSqrtAPI): + def executed_paddle_api(self, x): + return x.rsqrt() + + def executed_numpy_api(self, x): + return 1. / np.sqrt(x) + + +class TestDygraphInplaceRsqrtAPI(TestDygraphRsqrtAPI): + def executed_paddle_api(self, x): + return x.rsqrt_() + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_scale_op.py b/python/paddle/fluid/tests/unittests/test_scale_op.py index 052704659b6ed..c1ce032f50612 100644 --- a/python/paddle/fluid/tests/unittests/test_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_scale_op.py @@ -17,9 +17,11 @@ import unittest import numpy as np from op_test import OpTest +import paddle import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator +from paddle.static import Program, program_guard class TestScaleOp(OpTest): @@ -168,5 +170,45 @@ def test_scale_selected_rows_inplace(self): self.check_with_place(place, 'in', 'in') +class TestScaleApiStatic(unittest.TestCase): + def _executed_api(self, x, scale=1.0, bias=0.0): + return paddle.scale(x, scale, bias) + + def test_api(self): + paddle.enable_static() + input = np.random.random([2, 25]).astype("float32") + main_prog = Program() + with program_guard(main_prog, Program()): + x = paddle.static.data(name="x", shape=[2, 25], dtype="float32") + out = self._executed_api(x, scale=2.0, bias=3.0) + + exe = paddle.static.Executor(place=paddle.CPUPlace()) + out = exe.run(main_prog, feed={"x": input}, fetch_list=[out]) + self.assertEqual(np.array_equal(out[0], input * 2.0 + 3.0), True) + + +class TestScaleInplaceApiStatic(TestScaleApiStatic): + def _executed_api(self, x, scale=1.0, bias=0.0): + return x.scale_(scale, bias) + + +class TestScaleApiDygraph(unittest.TestCase): + def _executed_api(self, x, scale=1.0, bias=0.0): + return paddle.scale(x, scale, bias) + + def test_api(self): + paddle.disable_static() + input = np.random.random([2, 25]).astype("float32") + x = paddle.to_tensor(input) + out = self._executed_api(x, scale=2.0, bias=3.0) + self.assertEqual(np.array_equal(out.numpy(), input * 2.0 + 3.0), True) + paddle.enable_static() + + +class TestScaleInplaceApiDygraph(TestScaleApiDygraph): + def _executed_api(self, x, scale=1.0, bias=0.0): + return x.scale_(scale, bias) + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index 9001ba16b7ac2..d5dc6322522bb 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -16,7 +16,7 @@ from ...tensor.math import tanh # noqa: F401 from ...tensor.math import tanh_ # noqa: F401 -from ...tensor.manipulation import _print_warning_in_static_mode +from ...fluid.dygraph.inplace_utils import inplace_apis_in_dygraph_only from ...tensor.manipulation import chunk from ...tensor.math import multiply @@ -73,17 +73,13 @@ def elu(x, alpha=1.0, name=None): return out +@inplace_apis_in_dygraph_only def elu_(x, alpha=1.0, name=None): r""" Inplace version of ``elu`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_nn_cn_elu`. """ - - if in_dygraph_mode(): - return core.ops.elu_(x, 'alpha', alpha) - - _print_warning_in_static_mode("elu") - return elu(x, alpha, name) + return core.ops.elu_(x, 'alpha', alpha) def gelu(x, approximate=False, name=None): @@ -501,17 +497,13 @@ def relu(x, name=None): return out +@inplace_apis_in_dygraph_only def relu_(x, name=None): """ Inplace version of ``relu`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_nn_cn_relu`. """ - - if in_dygraph_mode(): - return core.ops.relu_(x) - - _print_warning_in_static_mode("relu") - return relu(x, name) + return core.ops.relu_(x) def log_sigmoid(x, name=None): @@ -912,21 +904,16 @@ def softmax(x, axis=-1, dtype=None, name=None): return outs_softmax +@inplace_apis_in_dygraph_only def softmax_(x, axis=-1, dtype=None, name=None): r""" Inplace version of ``softmax`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_nn_cn_softmax`. """ - if (dtype is not None) and (not isinstance(dtype, core.VarDesc.VarType)): dtype = convert_np_dtype_to_dtype_(dtype) use_cudnn = True - - if in_dygraph_mode(): - return core.ops.softmax_(x, 'axis', axis, 'use_cudnn', use_cudnn) - - _print_warning_in_static_mode("softmax") - return softmax(x, axis, dtype, name) + return core.ops.softmax_(x, 'axis', axis, 'use_cudnn', use_cudnn) def softplus(x, beta=1, threshold=20, name=None): diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py index c863f2b86a512..c8d80fc9bc68c 100755 --- a/python/paddle/tensor/__init__.py +++ b/python/paddle/tensor/__init__.py @@ -65,6 +65,7 @@ from .manipulation import expand_as # noqa: F401 from .manipulation import tile # noqa: F401 from .manipulation import flatten # noqa: F401 +from .manipulation import flatten_ # noqa: F401 from .manipulation import gather # noqa: F401 from .manipulation import gather_nd # noqa: F401 from .manipulation import reshape # noqa: F401 @@ -95,24 +96,32 @@ from .math import asin # noqa: F401 from .math import atan # noqa: F401 from .math import ceil # noqa: F401 +from .math import ceil_ # noqa: F401 from .math import cos # noqa: F401 from .math import tan # noqa: F401 from .math import cosh # noqa: F401 from .math import cumsum # noqa: F401 from .math import exp # noqa: F401 +from .math import exp_ # noqa: F401 from .math import floor # noqa: F401 +from .math import floor_ # noqa: F401 from .math import increment # noqa: F401 from .math import log # noqa: F401 from .math import multiplex # noqa: F401 from .math import pow # noqa: F401 from .math import reciprocal # noqa: F401 +from .math import reciprocal_ # noqa: F401 from .math import round # noqa: F401 +from .math import round_ # noqa: F401 from .math import rsqrt # noqa: F401 +from .math import rsqrt_ # noqa: F401 from .math import scale # noqa: F401 +from .math import scale_ # noqa: F401 from .math import sign # noqa: F401 from .math import sin # noqa: F401 from .math import sinh # noqa: F401 from .math import sqrt # noqa: F401 +from .math import sqrt_ # noqa: F401 from .math import square # noqa: F401 from .math import stanh # noqa: F401 from .math import sum # noqa: F401 @@ -131,7 +140,9 @@ from .math import floor_mod # noqa: F401 from .math import multiply # noqa: F401 from .math import add # noqa: F401 +from .math import add_ # noqa: F401 from .math import subtract # noqa: F401 +from .math import subtract_ # noqa: F401 from .math import atan # noqa: F401 from .math import logsumexp # noqa: F401 from .math import inverse # noqa: F401 @@ -141,6 +152,7 @@ from .math import erf # noqa: F401 from .math import addmm # noqa: F401 from .math import clip # noqa: F401 +from .math import clip_ # noqa: F401 from .math import trace # noqa: F401 from .math import kron # noqa: F401 from .math import isfinite # noqa: F401 @@ -202,11 +214,14 @@ 'asin', 'atan', 'ceil', + 'ceil_', 'cos', 'cosh', 'cumsum', 'exp', + 'exp_', 'floor', + 'floor_', 'increment', 'log', 'log2', @@ -217,13 +232,18 @@ 'pow', 'prod', 'reciprocal', + 'reciprocal_', 'round', + 'round_', 'rsqrt', + 'rsqrt_', 'scale', + 'scale_', 'sign', 'sin', 'sinh', 'sqrt', + 'sqrt_', 'square', 'stanh', 'sum', @@ -242,7 +262,9 @@ 'floor_mod', 'multiply', 'add', + 'add_', 'subtract', + 'subtract_', 'atan', 'logsumexp', 'inverse', @@ -250,6 +272,7 @@ 'erf', 'addmm', 'clip', + 'clip_', 'trace', 'kron', 'isfinite', @@ -277,6 +300,7 @@ 'broadcast_to', 'expand_as', 'flatten', + 'flatten_', 'gather', 'gather_nd', 'reshape', diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 1a5962042675d..97826f7d5f81d 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -31,18 +31,12 @@ from ..fluid.layers import scatter_nd # noqa: F401 from ..fluid.layers import shard_index # noqa: F401 from ..fluid import layers +from ..fluid.dygraph.inplace_utils import inplace_apis_in_dygraph_only import paddle -import warnings __all__ = [] -def _print_warning_in_static_mode(api_name): - warnings.warn( - "In static mode, {}_() is the same as {}() and does not perform inplace operation.". - format(api_name, api_name)) - - @dygraph_only def tolist(x): """ @@ -289,6 +283,36 @@ def flatten(x, start_axis=0, stop_axis=-1, name=None): return out +@inplace_apis_in_dygraph_only +def flatten_(x, start_axis=0, stop_axis=-1, name=None): + """ + Inplace version of ``flatten`` API, the output Tensor will be inplaced with input ``x``. + Please refer to :ref:`api_tensor_flatten`. + """ + if not (isinstance(x, Variable)): + raise ValueError("The input x should be a Tensor") + + x_dim = len(x.shape) + if not (isinstance(start_axis, int)) or ( + start_axis > x_dim - 1) or start_axis < -x_dim: + raise ValueError( + "The start_axis should be a int, and in range [-rank(x), rank(x))") + if not (isinstance(stop_axis, int)) or ( + stop_axis > x_dim - 1) or stop_axis < -x_dim: + raise ValueError( + "The stop_axis should be a int, and in range [-rank(x), rank(x))") + if start_axis < 0: + start_axis = start_axis + x_dim + if stop_axis < 0: + stop_axis = stop_axis + x_dim + if start_axis > stop_axis: + raise ValueError("The stop_axis should be larger than stat_axis") + + dy_out, _ = core.ops.flatten_contiguous_range_(x, 'start_axis', start_axis, + 'stop_axis', stop_axis) + return dy_out + + def roll(x, shifts, axis=None, name=None): """ Roll the `x` tensor along the given axis(axes). With specific 'shifts', Elements that @@ -582,6 +606,7 @@ def squeeze(x, axis=None, name=None): return layers.squeeze(x, axis, name) +@inplace_apis_in_dygraph_only def squeeze_(x, axis=None, name=None): """ Inplace version of ``squeeze`` API, the output Tensor will be inplaced with input ``x``. @@ -594,12 +619,8 @@ def squeeze_(x, axis=None, name=None): elif isinstance(axis, tuple): axis = list(axis) - if in_dygraph_mode(): - out, _ = core.ops.squeeze2_(x, 'axes', axis) - return out - - _print_warning_in_static_mode("squeeze") - return squeeze(x, axis, name) + out, _ = core.ops.squeeze2_(x, 'axes', axis) + return out def unique(x, @@ -775,26 +796,23 @@ def unsqueeze(x, axis, name=None): return layers.unsqueeze(x, axis, name) +@inplace_apis_in_dygraph_only def unsqueeze_(x, axis, name=None): """ Inplace version of ``unsqueeze`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_paddle_tensor_unsqueeze`. """ - if in_dygraph_mode(): - if isinstance(axis, int): - axis = [axis] - elif isinstance(axis, Variable): - axis = axis.numpy().tolist() - elif isinstance(axis, (list, tuple)): - axis = [ - item.numpy().item(0) if isinstance(item, Variable) else item - for item in axis - ] - out, _ = core.ops.unsqueeze2_(x, 'axes', axis) - return out - - _print_warning_in_static_mode("unsqueeze") - return unsqueeze(x, axis, name) + if isinstance(axis, int): + axis = [axis] + elif isinstance(axis, Variable): + axis = axis.numpy().tolist() + elif isinstance(axis, (list, tuple)): + axis = [ + item.numpy().item(0) if isinstance(item, Variable) else item + for item in axis + ] + out, _ = core.ops.unsqueeze2_(x, 'axes', axis) + return out def gather(x, index, axis=None, name=None): @@ -1023,16 +1041,13 @@ def scatter(x, index, updates, overwrite=True, name=None): return out +@inplace_apis_in_dygraph_only def scatter_(x, index, updates, overwrite=True, name=None): """ Inplace version of ``scatter`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_paddle_tensor_scatter`. """ - if in_dygraph_mode(): - return core.ops.scatter_(x, index, updates, 'overwrite', overwrite) - - _print_warning_in_static_mode("scatter") - return scatter(x, index, updates, overwrite, name) + return core.ops.scatter_(x, index, updates, 'overwrite', overwrite) def scatter_nd_add(x, index, updates, name=None): @@ -1555,26 +1570,23 @@ def reshape(x, shape, name=None): return paddle.fluid.layers.reshape(x=x, shape=shape, name=name) +@inplace_apis_in_dygraph_only def reshape_(x, shape, name=None): """ Inplace version of ``reshape`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_paddle_tensor_reshape`. """ - if in_dygraph_mode(): - if isinstance(shape, (list, tuple)): - shape = [ - item.numpy().item(0) if isinstance(item, Variable) else item - for item in shape - ] - out, _ = core.ops.reshape2_(x, None, 'shape', shape) - return out - elif isinstance(shape, Variable): - shape.stop_gradient = True - out, _ = core.ops.reshape2_(x, shape) - return out - - _print_warning_in_static_mode("reshape") - return reshape(x, shape, name) + if isinstance(shape, (list, tuple)): + shape = [ + item.numpy().item(0) if isinstance(item, Variable) else item + for item in shape + ] + out, _ = core.ops.reshape2_(x, None, 'shape', shape) + return out + elif isinstance(shape, Variable): + shape.stop_gradient = True + out, _ = core.ops.reshape2_(x, shape) + return out def gather_nd(x, index, name=None): diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index 84c67a9ae8d9d..23addcb7e3f4e 100755 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -30,7 +30,7 @@ from ..fluid.layer_helper import LayerHelper from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype from ..fluid.layers.layer_function_generator import _generate_doc_string_, generate_activation_fn, generate_layer_fn -from .manipulation import _print_warning_in_static_mode +from ..fluid.dygraph.inplace_utils import inplace_apis_in_dygraph_only # TODO: define math functions # yapf: disable @@ -38,22 +38,29 @@ from ..fluid.layers import acos # noqa: F401 from ..fluid.layers import asin # noqa: F401 from ..fluid.layers import ceil # noqa: F401 +from ..fluid.layers import ceil_ # noqa: F401 from ..fluid.layers import cos # noqa: F401 from ..fluid.layers import tan # noqa: F401 from ..fluid.layers import sinh # noqa: F401 from ..fluid.layers import cosh # noqa: F401 from ..fluid.layers import exp # noqa: F401 +from ..fluid.layers import exp_ # noqa: F401 from ..fluid.layers import floor # noqa: F401 +from ..fluid.layers import floor_ # noqa: F401 from ..fluid.layers import log # noqa: F401 from ..fluid.layers import reciprocal # noqa: F401 +from ..fluid.layers import reciprocal_ # noqa: F401 from ..fluid.layers import round # noqa: F401 +from ..fluid.layers import round_ # noqa: F401 from ..fluid.layers import rsqrt # noqa: F401 +from ..fluid.layers import rsqrt_ # noqa: F401 from ..fluid.layers import scale # noqa: F401 from ..fluid.layers import square # noqa: F401 from ..fluid.layers import stanh # noqa: F401 from ..fluid.layers import atan # noqa: F401 from ..fluid.layers import erf # noqa: F401 from ..fluid.layers import sqrt # noqa: F401 +from ..fluid.layers import sqrt_ # noqa: F401 from ..fluid.layers import sin # noqa: F401 from ..fluid.layers import multiplex # noqa: F401 @@ -74,6 +81,19 @@ VarDesc.VarType.FP64, ] + +@inplace_apis_in_dygraph_only +def scale_(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): + """ + Inplace version of ``scale`` API, the output Tensor will be inplaced with input ``x``. + Please refer to :ref:`api_tensor_scale`. + """ + _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale + return core.ops.scale_(x, 'scale', + float(_scale), 'bias', + float(bias), 'bias_after_scale', bias_after_scale) + + def pow(x, y, name=None): """ Compute the power of tensor elements. The equation is: @@ -221,6 +241,24 @@ def add(x, y, name=None): return _elementwise_op(LayerHelper(op_type, **locals())) +@inplace_apis_in_dygraph_only +def add_(x, y, name=None): + """ + Inplace version of ``add`` API, the output Tensor will be inplaced with input ``x``. + Please refer to :ref:`api_tensor_add`. + """ + op_type = 'elementwise_add_' + axis = -1 + + out_shape = broadcast_shape(x.shape, y.shape) + if out_shape != x.shape: + raise ValueError("The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format(out_shape, x.shape)) + + out = _elementwise_op_in_dygraph( + x, y, axis=axis, op_name=op_type) + return out + + def subtract(x, y, name=None): """ Substract two tensors element-wise. The equation is: @@ -282,6 +320,24 @@ def subtract(x, y, name=None): return _elementwise_op(LayerHelper(op_type, **locals())) +@inplace_apis_in_dygraph_only +def subtract_(x, y, name=None): + """ + Inplace version of ``subtract`` API, the output Tensor will be inplaced with input ``x``. + Please refer to :ref:`api_tensor_subtract`. + """ + axis = -1 + act = None + + out_shape = broadcast_shape(x.shape, y.shape) + if out_shape != x.shape: + raise ValueError("The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format(out_shape, x.shape)) + + out = _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_sub_') + return out + + def divide(x, y, name=None): """ Divide two tensors element-wise. The equation is: @@ -1489,6 +1545,24 @@ def clip(x, min=None, max=None, name=None): return output +@inplace_apis_in_dygraph_only +def clip_(x, min=None, max=None, name=None): + """ + Inplace version of ``clip`` API, the output Tensor will be inplaced with input ``x``. + Please refer to :ref:`api_tensor_clip`. + """ + fmin = float(np.finfo(np.float32).min) + fmax = float(np.finfo(np.float32).max) + if isinstance(min, Variable): + min = min.numpy().item(0) + if isinstance(max, Variable): + max = max.numpy().item(0) + min = fmin if min is None else min + max = fmax if max is None else max + return core.ops.clip_(x, "min", min, "max", max) + + + def trace(x, offset=0, axis1=0, axis2=1, name=None): """ **trace** @@ -1908,16 +1982,14 @@ def tanh(x, name=None): helper.append_op(type='tanh', inputs={'X': x}, outputs={'Out': out}) return out +@inplace_apis_in_dygraph_only def tanh_(x, name=None): r""" Inplace version of ``tanh`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_tensor_tanh`. """ - if in_dygraph_mode(): - return core.ops.tanh_(x) + return core.ops.tanh_(x) - _print_warning_in_static_mode("tanh") - return tanh(x, name) def increment(x, value=1.0, name=None): """ diff --git a/tools/wlist.json b/tools/wlist.json index cd9f2a7ca661e..5a83a9ee47004 100644 --- a/tools/wlist.json +++ b/tools/wlist.json @@ -34,6 +34,10 @@ "name":"reshape_", "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" }, + { + "name":"flatten_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, { "name":"scatter_", "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" @@ -53,6 +57,50 @@ { "name":"tanh_", "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"ceil_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"floor_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"exp_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"reciprocal_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"round_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"sqrt_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"rsqrt_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"clip_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"scale_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"subtract_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" + }, + { + "name":"add_", + "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" } ], "wlist_temp_api":[ From 09adf20fc8857b5ca5e296f6706c6deba1b35d50 Mon Sep 17 00:00:00 2001 From: XiangGao Date: Fri, 30 Apr 2021 18:37:47 +0800 Subject: [PATCH 023/156] add flag to check_kernel launch (#32692) (#32709) --- paddle/fluid/framework/op_registry.h | 13 +++++++++---- paddle/fluid/platform/flags.cc | 13 +++++++++++++ paddle/fluid/pybind/global_value_getter_setter.cc | 3 ++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/framework/op_registry.h b/paddle/fluid/framework/op_registry.h index 9f0dc50774add..593d4d839fa91 100644 --- a/paddle/fluid/framework/op_registry.h +++ b/paddle/fluid/framework/op_registry.h @@ -25,7 +25,8 @@ limitations under the License. */ #include #define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h -#include "glog/logging.h" // For VLOG() +#include "gflags/gflags.h" +#include "glog/logging.h" // For VLOG() #include "paddle/fluid/framework/attribute.h" #include "paddle/fluid/framework/details/op_registry.h" #include "paddle/fluid/framework/grad_op_desc_maker.h" @@ -67,6 +68,8 @@ class Version; } // namespace framework } // namespace paddle +DECLARE_bool(check_kernel_launch); + namespace paddle { namespace framework { @@ -135,14 +138,16 @@ class OpRegistry { }; template -inline void CheckKernelLaunch(const char* op_type){}; +inline void CheckKernelLaunch(const char* op_type) {} #ifdef PADDLE_WITH_CUDA template <> inline void CheckKernelLaunch<::paddle::platform::CUDAPlace>( const char* op_type) { - PADDLE_ENFORCE_CUDA_LAUNCH_SUCCESS(op_type); -}; + if (FLAGS_check_kernel_launch) { + PADDLE_ENFORCE_CUDA_LAUNCH_SUCCESS(op_type); + } +} #endif template diff --git a/paddle/fluid/platform/flags.cc b/paddle/fluid/platform/flags.cc index 83b9544d23267..1d76c2ea584b7 100644 --- a/paddle/fluid/platform/flags.cc +++ b/paddle/fluid/platform/flags.cc @@ -578,6 +578,19 @@ DEFINE_string(tracer_mkldnn_ops_on, "", DEFINE_string(tracer_mkldnn_ops_off, "", "List of OneDNN operation types to be turned off"); +/** + * Debug related FLAG + * Name: check_kernel_launch + * Since Version: 2.1.0 + * Value Range: bool, default=false + * Example: + * Note: Check kernel launch status after every kernel compute. + */ +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) +DEFINE_bool(check_kernel_launch, false, + "Check kernel launch status after every kernel compute"); +#endif + /** * CUDNN related FLAG * Name: conv2d_disable_cudnn diff --git a/paddle/fluid/pybind/global_value_getter_setter.cc b/paddle/fluid/pybind/global_value_getter_setter.cc index bc8d1e5b40585..4824a34e843bb 100644 --- a/paddle/fluid/pybind/global_value_getter_setter.cc +++ b/paddle/fluid/pybind/global_value_getter_setter.cc @@ -41,6 +41,7 @@ DECLARE_int32(multiple_of_cupti_buffer_size); DECLARE_bool(reader_queue_speed_test_mode); DECLARE_int32(call_stack_level); DECLARE_bool(sort_sum_gradient); +DECLARE_bool(check_kernel_launch); // device management DECLARE_int32(paddle_num_threads); // executor @@ -376,7 +377,7 @@ static void RegisterGlobalVarGetterSetter() { FLAGS_fraction_of_gpu_memory_to_use, FLAGS_initial_gpu_memory_in_mb, FLAGS_reallocate_gpu_memory_in_mb, FLAGS_enable_cublas_tensor_op_math, FLAGS_selected_gpus, FLAGS_sync_nccl_allreduce, - FLAGS_conv2d_disable_cudnn); + FLAGS_conv2d_disable_cudnn, FLAGS_check_kernel_launch); #endif #ifdef PADDLE_WITH_XPU REGISTER_PUBLIC_GLOBAL_VAR(FLAGS_selected_xpus); From 2c1ed9b8d4e8392c37f32d360f85ccb44d20156f Mon Sep 17 00:00:00 2001 From: liuyuhui Date: Sat, 1 May 2021 14:13:00 +0800 Subject: [PATCH 024/156] [Kunlun]fix multi xpu dygraph hang, test=kunlun (#32662) (#32696) --- paddle/fluid/imperative/reducer.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/imperative/reducer.cc b/paddle/fluid/imperative/reducer.cc index a92704ce447dc..bf479e0d797ca 100644 --- a/paddle/fluid/imperative/reducer.cc +++ b/paddle/fluid/imperative/reducer.cc @@ -762,10 +762,11 @@ void Reducer::MarkGroupReady(size_t group_index) { // TODO(liuyuhui): Add try catch to deal with exception later, // otherwise the main thread will continue to run when an exception is // thrown in comm_pool_. - comm_pool_->enqueue([&] { + auto next_group = next_group_; + comm_pool_->enqueue([this, run_order, next_group, &group] { auto dev_id = BOOST_GET_CONST(platform::XPUPlace, place_).device; platform::SetXPUDeviceId(dev_id); - FusedAllReduceSchedule(run_order, group, next_group_); + FusedAllReduceSchedule(run_order, group, next_group); { std::lock_guard lock(mutex_); comm_op_count_ -= 1; // lock From 6a1957e7482fd7575d1b0ffa4f5018c467d4636c Mon Sep 17 00:00:00 2001 From: Baibaifan <39549453+Baibaifan@users.noreply.github.com> Date: Sat, 1 May 2021 14:13:19 +0800 Subject: [PATCH 025/156] slove develop bugs (#32560) (#32684) --- paddle/fluid/operators/collective/c_sync_comm_stream_op.cc | 2 -- paddle/fluid/pybind/ascend_wrapper_py.cc | 2 ++ python/paddle/distributed/fleet/launch.py | 4 ++-- python/paddle/distributed/fleet/launch_utils.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc index e6f6bf5345619..772122bb58d60 100644 --- a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc +++ b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc @@ -63,7 +63,6 @@ class CSyncCommStreamCudaKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { auto place = ctx.GetPlace(); #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) - int ring_id = ctx.Attr("ring_id"); auto stream = platform::NCCLCommContext::Instance().Get(ring_id, place)->stream(); @@ -75,7 +74,6 @@ class CSyncCommStreamCudaKernel : public framework::OpKernel { #endif #elif defined(PADDLE_WITH_ASCEND_CL) - auto place = ctx.GetPlace(); PADDLE_ENFORCE_EQ(is_npu_place(place), true, platform::errors::PreconditionNotMet( "Sync stream op can run on npu place only for now.")); diff --git a/paddle/fluid/pybind/ascend_wrapper_py.cc b/paddle/fluid/pybind/ascend_wrapper_py.cc index 9a1fa1d7704c2..43725f7dc0f73 100644 --- a/paddle/fluid/pybind/ascend_wrapper_py.cc +++ b/paddle/fluid/pybind/ascend_wrapper_py.cc @@ -108,12 +108,14 @@ enum AttrType { AT_NAMEATTR }; +#ifdef PADDLE_WITH_ASCEND void BindAscendDevice(py::module *m) { py::class_(*m, "NPUDevice") .def_static( "get_device_count", static_cast(&platform::ascend::NPUDevice::GetDeviceCount)); } +#endif void BindAscendGraph(py::module *m) { m->def("ge_initialize", &ge_initialize, "GEInitialize"); diff --git a/python/paddle/distributed/fleet/launch.py b/python/paddle/distributed/fleet/launch.py index 89ca7e1961331..69c5b325d182d 100644 --- a/python/paddle/distributed/fleet/launch.py +++ b/python/paddle/distributed/fleet/launch.py @@ -325,8 +325,8 @@ def which_distributed_mode(args): if fluid.core.is_compiled_with_cuda(): accelerators = fluid.core.get_cuda_device_count() - elif fluid.core.is_compiled_with_ascend(): - accelerators = fluid.core.NPUDevice.get_device_count() + elif fluid.core.is_compiled_with_npu(): + accelerators = fluid.core.get_npu_device_count() elif fluid.core.is_compiled_with_xpu(): accelerators = fluid.core.get_xpu_device_count() else: diff --git a/python/paddle/distributed/fleet/launch_utils.py b/python/paddle/distributed/fleet/launch_utils.py index b4d5c58abbf2e..be7ad257ccb99 100644 --- a/python/paddle/distributed/fleet/launch_utils.py +++ b/python/paddle/distributed/fleet/launch_utils.py @@ -653,8 +653,8 @@ def get_xpus(xpus): def get_device_mode(): - if fluid.core.is_compiled_with_ascend() and \ - fluid.core.NPUDevice.get_device_count() > 0: + if fluid.core.is_compiled_with_npu() and \ + fluid.core.get_npu_device_count() > 0: print("launch train in ascend npu mode!") return DeviceMode.ASCEND_NPU From 4593597d6a135eafb03521337eea0f7246f07c6d Mon Sep 17 00:00:00 2001 From: Baibaifan <39549453+Baibaifan@users.noreply.github.com> Date: Tue, 4 May 2021 16:07:44 +0800 Subject: [PATCH 026/156] add_c_sync_npu_kernel (#32687) (#32723) --- paddle/fluid/operators/collective/c_sync_calc_stream_op.cc | 7 ++++--- .../operators/collective/c_sync_calc_stream_op_npu_test.cc | 2 +- paddle/fluid/operators/collective/c_sync_comm_stream_op.cc | 7 ++++--- .../operators/collective/c_sync_comm_stream_op_npu_test.cc | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc b/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc index 83da712bee908..71ab25a7b0ff8 100644 --- a/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc +++ b/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc @@ -46,7 +46,7 @@ Call calculation stream synchronization. }; template -class CSyncCalcStreamCudaKernel : public framework::OpKernel { +class CSyncCalcStreamKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { #if (defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)) && !defined(_WIN32) @@ -86,5 +86,6 @@ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(c_sync_calc_stream, ops::CSyncCalcStreamOp, ops::CSyncCalcStreamOpMaker); -REGISTER_OP_CUDA_KERNEL(c_sync_calc_stream, - ops::CSyncCalcStreamCudaKernel); +REGISTER_OP_CUDA_KERNEL(c_sync_calc_stream, ops::CSyncCalcStreamKernel); + +REGISTER_OP_NPU_KERNEL(c_sync_calc_stream, ops::CSyncCalcStreamKernel); diff --git a/paddle/fluid/operators/collective/c_sync_calc_stream_op_npu_test.cc b/paddle/fluid/operators/collective/c_sync_calc_stream_op_npu_test.cc index 4b1f7bb340178..45613715b8260 100644 --- a/paddle/fluid/operators/collective/c_sync_calc_stream_op_npu_test.cc +++ b/paddle/fluid/operators/collective/c_sync_calc_stream_op_npu_test.cc @@ -35,7 +35,7 @@ namespace m = paddle::operators::math; USE_OP(elementwise_add); USE_OP_DEVICE_KERNEL(elementwise_add, NPU); -USE_NO_KERNEL_OP(c_sync_calc_stream); +USE_OP_DEVICE_KERNEL(c_sync_calc_stream, NPU); template void Compare(f::Scope* scope, const p::DeviceContext& ctx) { diff --git a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc index 772122bb58d60..71fda2cd01c8d 100644 --- a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc +++ b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc @@ -58,7 +58,7 @@ Call communication stream synchronization. }; template -class CSyncCommStreamCudaKernel : public framework::OpKernel { +class CSyncCommStreamKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { auto place = ctx.GetPlace(); @@ -97,5 +97,6 @@ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(c_sync_comm_stream, ops::CSyncCommStreamOp, ops::CSyncCommStreamOpMaker); -REGISTER_OP_CUDA_KERNEL(c_sync_comm_stream, - ops::CSyncCommStreamCudaKernel); +REGISTER_OP_CUDA_KERNEL(c_sync_comm_stream, ops::CSyncCommStreamKernel); + +REGISTER_OP_NPU_KERNEL(c_sync_comm_stream, ops::CSyncCommStreamKernel); diff --git a/paddle/fluid/operators/collective/c_sync_comm_stream_op_npu_test.cc b/paddle/fluid/operators/collective/c_sync_comm_stream_op_npu_test.cc index 3915ec4fa35e8..6c5a6db61483d 100644 --- a/paddle/fluid/operators/collective/c_sync_comm_stream_op_npu_test.cc +++ b/paddle/fluid/operators/collective/c_sync_comm_stream_op_npu_test.cc @@ -43,7 +43,7 @@ namespace p = paddle::platform; namespace m = paddle::operators::math; USE_OP(c_broadcast); -USE_NO_KERNEL_OP(c_sync_comm_stream); +USE_OP_DEVICE_KERNEL(c_sync_comm_stream, NPU); USE_NO_KERNEL_OP(c_gen_hccl_id); USE_NO_KERNEL_OP(c_comm_init_hccl); USE_OP_DEVICE_KERNEL(c_broadcast, NPU); From 6b86e966c43578f07432fb6e35d78fbe878320b8 Mon Sep 17 00:00:00 2001 From: lilong12 Date: Wed, 5 May 2021 12:52:56 +0800 Subject: [PATCH 027/156] Fix the bug in pipeline for dygraph mode (#32716) (#32728) * update, test=develop --- .../parallel_layers/pp_layers.py | 1 - .../fleet/meta_parallel/pipeline_parallel.py | 342 ++++++++++-------- .../fleet/meta_parallel/pp_utils/utils.py | 43 ++- 3 files changed, 231 insertions(+), 155 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py index 669ed032a3443..a9704e38f3fa7 100644 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py @@ -108,7 +108,6 @@ def __init__(self, # construct layer self.run_function = [] self._build_layer() - self.to(paddle.CUDAPlace(self.device_id)) def _segment_network(self, seg_method): logger.info("start segment network..") diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 98a82f2b79856..11180054afbfc 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -22,15 +22,11 @@ import paddle import paddle.fluid as fluid from .meta_parallel_base import MetaParallelBase -from .pp_utils.utils import get_tensor_bytes +from .pp_utils.utils import get_tensor_bytes, is_float_tensor from .pp_utils import utils from .parallel_layers.pp_layers import PipelineLayer - -FLOAT_TYPES = [ - paddle.float16, - paddle.float32, - paddle.float64, -] +from ..utils.hybrid_parallel_util import * +from ..utils.log_util import logger class PipelineParallel(MetaParallelBase): @@ -46,20 +42,18 @@ def __init__(self, layers, hcg, strategy): 'inputs': [], 'labels': [], 'outputs': [], - 'backward_tensors': [], } + self.recv_cache = None self.grad_tensors = None - self.meta_buffer = None - self.send_meta = True - self.first_gradient_send = True self.current_loss = paddle.to_tensor(0.0) self.total_loss = None - def _prepare_for_model(self): + self.use_amp = self._strategy.amp + self.init_loss_scaling = self._strategy.amp_configs['init_loss_scaling'] self.micro_batch_size = self._strategy.pipeline_configs[ 'micro_batch_size'] self.accumulate_steps = self._strategy.pipeline_configs[ @@ -69,9 +63,17 @@ def _prepare_for_model(self): self.stage_id = self._hcg.get_stage_id() self.prev_stage_id = self.stage_id - 1 self.next_stage_id = self.stage_id + 1 - self._layers = PipelineLayer( - layers=self._layers, num_stages=self.num_stages) - #TODO: init process group + self.pp_group = self._hcg.get_pipe_parallel_group() + logger.info("Pipeline Info -- num_stages: {}, stage_id: {}".format( + self.num_stages, self.stage_id)) + + if self.use_model_parallel: + logger.info("start broadcast mp parameters") + broadcast_mp_parameters(self._layers, self._hcg) + + if self.use_data_parallel: + logger.info("start broadcast mp parameters") + broadcast_dp_parameters(self._layers, self._hcg) def _allocate_caches(self, num_caches): if self.num_caches >= num_caches: @@ -82,19 +84,19 @@ def _allocate_caches(self, num_caches): for key in self.caches: self.caches[key].extend([None] * num) - def train_batch(self, data_iter, optimizer): + def train_batch(self, data, optimizer): self.optimizer = optimizer assert fluid.framework._dygraph_tracer()._has_grad, ( 'Please enable the generation of gradients.') if self.stage_id == 0 or self.stage_id == self.num_stages - 1: - assert data_iter, ( + assert data, ( "For the first and the last stage, the data_iter must be set.") else: - assert data_iter is None, ( + assert data is None, ( "For pipe stages other than the first and the last one, " "the data_iter must be None.") - self.data_iter = data_iter + self.data = data self._layers.train() self.total_loss = None @@ -104,39 +106,24 @@ def train_batch(self, data_iter, optimizer): return self.total_loss def _train(self, minibatch_cmds): - self._allocate_caches(self.num_stages) - for microbatch_cmds in minibatch_cmds: - for cmd in microbatch_cmds: - if type(cmd) not in self._COMMAND_MAP: - #FIXME: - continue - + self._allocate_caches(self.accumulate_steps) + for micro_cmds in minibatch_cmds: + for cmd in micro_cmds: + assert type(cmd) in self._COMMAND_MAP, "unknow cmd: {}".format( + type(cmd)) self._apply_cmd = MethodType(self._COMMAND_MAP[type(cmd)], self) self._apply_cmd(**cmd.kwargs) def _allreduce_grads(self): - self._modifying_grad = True - assert self.use_data_parallel <= 1, ("Do not support data parallel " - "with pipeline parallel now.") - self._modifying_grad = False - - def _get_data(self): - if self.use_model_parallel: - mp_rank = self._hcg.get_model_parallel_rank() - else: - mp_rank = 0 - - data = None - - # mp rank 0 loads the data and broadcat it to others. - if mp_rank == 0: - data = next(self.data_iter) - if self.use_model_parallel: - data = paddle.distributed.broadcast( - data, group=self._hcg.get_model_parallel_group()) - return data + if not self.use_data_parallel: return + fused_allreduce_gradients(list(self._layers.parameters()), self._hcg) def _forward(self, cache_id): + # load data + self._load_micro_batch(cache_id) + if self.stage_id != 0: + self._recv_activations(cache_id) + if isinstance(self.caches['inputs'][cache_id], tuple): inputs = tuple(t.clone() for t in self.caches['inputs'][cache_id]) else: @@ -144,9 +131,13 @@ def _forward(self, cache_id): self._clear_grads(inputs) outputs = self._layers.forward(inputs) - self.caches['outputs'][cache_id] = outputs + if self.stage_id == self.num_stages - 1: + if self._layers._loss_fn is not None: + labels = self.caches['labels'][cache_id] + outputs = self._layers._loss_fn(outputs, labels) + if self.stage_id == self.num_stages - 1: self.current_loss = outputs if isinstance(self.current_loss, paddle.Tensor): @@ -160,18 +151,28 @@ def _forward(self, cache_id): ] for idx, v in enumerate(self.current_loss): self.total_loss[idx] += v.detach() + if self.use_data_parallel: + self.current_loss = self.current_loss / self._hcg.get_data_parallel_world_size( + ) + if self.accumulate_steps > 1: + self.current_loss = self.current_loss / self.accumulate_steps + self.caches['outputs'][cache_id] = self.current_loss.clone() + else: + self._send_activations(cache_id) def _backward(self, cache_id): assert self.optimizer is not None if self.stage_id == self.num_stages - 1: - paddle.autograd.backward(self.current_loss) + paddle.autograd.backward(self.caches['outputs'][cache_id]) + self._send_gradients(cache_id) return + self._recv_gradients(cache_id) outputs = self.caches['outputs'][cache_id] grad_tensors = self.grad_tensors if isinstance(outputs, tuple): - out_tensors = [t for t in outputs if t.dtype in FLOAT_TYPES] + out_tensors = [t for t in outputs if is_float_tensor(t)] assert len(out_tensors) == len(grad_tensors) paddle.autograd.backward( tensors=out_tensors, grad_tensors=grad_tensors) @@ -179,41 +180,76 @@ def _backward(self, cache_id): paddle.autograd.backward( tensors=[outputs], grad_tensors=[grad_tensors]) - self.caches['outputs'][cache_id] = None grad_tensors = None + if self.stage_id != 0: self._send_gradients(cache_id) + self.caches['outputs'][cache_id] = None + #self.caches['backward_tensors'][cache_id] = None + + def _get_data(self): + if self.use_model_parallel: + mp_rank = self._hcg.get_model_parallel_rank() + else: + mp_rank = 0 + + # mp rank 0 loads the data and broadcat it to others. + data = self.data + if self.use_model_parallel and (self.stage_id == 0 or + self.stage_id == self.num_stages - 1): + assert isinstance(data, (tuple, paddle.Tensor)) + if isinstance(data, paddle.Tensor): + paddle.distributed.broadcast( + data, + src=self._hcg.get_model_parallel_group_src_rank(), + group=self._hcg.get_model_parallel_group()) + else: + data = [] + for d in self.data: + assert isinstance(d, paddle.Tensor) + paddle.distributed.broadcast( + d, + src=self._hcg.get_model_parallel_group_src_rank(), + group=self._hcg.get_model_parallel_group()) + data.append(d) + data = tuple(data) + return data def _load_micro_batch(self, cache_id): inputs = self._get_data() if self.stage_id == 0: data = None - if isinstance(inputs[0], paddle.Tensor): + #if isinstance(inputs[0], paddle.Tensor): + if len(inputs) == 1: + assert isinstance(inputs[0], paddle.Tensor) data = inputs[0].clone().detach() - data.stop_gradient = data.dtype == paddle.float32 + #data.stop_gradient = not is_float_tensor(data) + data.stop_gradient = True else: - assert isinstance(inputs[0], tuple) - # Assume list or tuple + assert isinstance(inputs, tuple) data = [] - for d in inputs[0]: + for d in inputs: assert isinstance(d, paddle.Tensor) - d = d.clone().detach() - d.stop_gradient = d.dtype == paddle.float32 - loaded.append(d) + i = d.clone().detach() + #i.stop_gradient = not is_float_tensor(i) + i.stop_gradient = True + data.append(i) data = tuple(data) self.caches['inputs'][cache_id] = data if self.stage_id == self.num_stages - 1: - label = None - if isinstance(inputs[1], paddle.Tensor): - label = inputs[1] - elif isinstance(data[1], tuple): - label = [] - for l in inputs[1]: - assert isinstance(l, paddle.Tensor) - l = l.detach() - label.append(l) - label = tuple(label) - self.caches['labels'][cache_id] = label + labels = None + #if isinstance(inputs[1], paddle.Tensor): + if len(inputs) == 1: + assert isinstance(inputs[0], paddle.Tensor) + labels = inputs[0] + elif isinstance(inputs, tuple): + labels = [] + for label in inputs: + assert isinstance(label, paddle.Tensor) + label = label.detach() + labels.append(label) + labels = tuple(labels) + self.caches['labels'][cache_id] = labels def _send_meta(self, data, peer): """ @@ -225,54 +261,67 @@ def _send_meta(self, data, peer): """ if isinstance(data, paddle.Tensor): tensor_type = paddle.to_tensor([0]) - paddle.distributed.send(tensor_type, peer) + paddle.distributed.send( + tensor_type, peer, use_calc_stream=True, group=self.pp_group) dims = paddle.to_tensor(len(data.shape)) - paddle.distributed.send(dims, peer) + paddle.distributed.send( + dims, peer, use_calc_stream=True, group=self.pp_group) shape = paddle.to_tensor(data.shape) - paddle.distributed.send(shape, peer) + paddle.distributed.send( + shape, peer, use_calc_stream=True, group=self.pp_group) elif isinstance(data, tuple): tensor_type = paddle.to_tensor([1]) - paddle.distributed.send(tensor_type, peer) + paddle.distributed.send( + tensor_type, peer, use_calc_stream=True, group=self.pp_group) nums = paddle.to_tensor(len(data)) - paddle.distributed.send(nums, peer) + paddle.distributed.send( + nums, peer, use_calc_stream=True, group=self.pp_group) for idx, d in enumerate(data): assert isinstance(d, paddle.Tensor) dims = paddle.to_tensor(len(d.shape)) - paddle.distributed.send(dims, peer) + paddle.distributed.send( + dims, peer, use_calc_stream=True, group=self.pp_group) shape = paddle.to_tensor(d.shape) - paddle.distributed.send(shape, peer) + paddle.distributed.send( + shape, peer, use_calc_stream=True, group=self.pp_group) def _recv_meta(self, peer): tensor_type = paddle.to_tensor([0]) - paddle.distributed.recv(tensor_type, peer) + paddle.distributed.recv( + tensor_type, peer, use_calc_stream=True, group=self.pp_group) tensor_type = tensor_type.numpy()[0] if tensor_type == 0: dims = paddle.to_tensor([0]) - paddle.distributed.recv(dims, peer) + paddle.distributed.recv( + dims, peer, use_calc_stream=True, group=self.pp_group) dims = dims.numpy()[0] shape = paddle.to_tensor([0] * dims) - paddle.distributed.recv(shape, peer) + paddle.distributed.recv( + shape, peer, use_calc_stream=True, group=self.pp_group) shape = shape.numpy().tolist() return self._allocate_buffer( shape, dtype="float32", num_caches=1)[0] elif tensor_type == 1: num = paddle.to_tensor([0]) - paddle.distributed.recv(num, peer) + paddle.distributed.recv( + num, peer, use_calc_stream=True, group=self.pp_group) num = num.numpy()[0] shapes = [] for i in range(num): dims = paddle.to_tensor([0]) - paddle.distributed.recv(dims, peer) + paddle.distributed.recv( + dims, peer, use_calc_stream=True, group=self.pp_group) dims = dims.numpy()[0] shape = paddle.to_tensor([0] * dims) - paddle.distributed.recv(shape, peer) + paddle.distributed.recv( + shape, peer, use_calc_stream=True, group=self.pp_group) shapes.append(shape.numpy().tolist()) dtypes = ["float32"] * len(shapes) - caches = self._allocate_buffers(shapes, dtypes, num_buffers=1)[0] - buffers = tuple(buffers) - return buffers + caches = self._allocate_buffers(shapes, dtypes, num_caches=1)[0] + caches = tuple(caches) + return caches def _send_activations(self, cache_id): outputs = self.caches['outputs'][cache_id] @@ -282,10 +331,18 @@ def _send_activations(self, cache_id): self._send_meta(outputs, self.next_stage_id) if isinstance(outputs, paddle.Tensor): - paddle.distributed.send(outputs, self.next_stage_id) + paddle.distributed.send( + outputs, + self.next_stage_id, + use_calc_stream=True, + group=self.pp_group) elif isinstance(outputs, tuple): for output in outputs: - paddle.distributed.send(output, self.next_stage_id) + paddle.distributed.send( + output, + self.next_stage_id, + use_calc_stream=True, + group=self.pp_group) def _send_gradients(self, cache_id): inputs = self.caches['inputs'][cache_id] @@ -293,15 +350,22 @@ def _send_gradients(self, cache_id): if isinstance(inputs, paddle.Tensor): assert inputs.grad is not None paddle.distributed.send( - paddle.to_tensor(inputs.grad), self.prev_stage_id) + paddle.to_tensor(inputs.grad), + self.prev_stage_id, + use_calc_stream=True, + group=self.pp_group) else: for idx, d in enumerate(inputs): # Skip tensors that will not produce a grad - if not d.dtype in FLOAT_TYPES: + if not is_float_tensor(d): assert d.grad is None continue assert d.grad is not None - paddle.distributed.send(d.grad, self.prev_stage_id) + paddle.distributed.send( + d.grad, + self.prev_stage_id, + use_calc_stream=True, + group=self.pp_group) self.caches['inputs'][cache_id] = None def _recv_activations(self, cache_id): @@ -312,22 +376,30 @@ def _recv_activations(self, cache_id): self.recv_cache = self._recv_meta(self.prev_stage_id) if isinstance(self.recv_cache, paddle.Tensor): - paddle.distributed.recv(self.recv_cache, self.prev_stage_id) + paddle.distributed.recv( + self.recv_cache, + self.prev_stage_id, + use_calc_stream=True, + group=self.pp_group) inputs = self.recv_cache.clone().detach() - inputs.stop_gradient = inputs.dtype not in FLOAT_TYPES + inputs.stop_gradient = not is_float_tensor(inputs) else: assert isinstance(self.recv_cache, tuple) inputs = [None] * len(self.recv_cache) for idx, d in enumerate(self.recv_cache): assert isinstance(d, paddle.Tensor) - paddle.distributed.recv(d, self.prev_stage_id) + paddle.distributed.recv( + d, + self.prev_stage_id, + use_calc_stream=True, + group=self.pp_group) inputs[idx] = d.clone().detach() inputs = tuple(inputs) for d in inputs: - d.stop_gradient = d.dtype not in FLOAT_TYPES + d.stop_gradient = not is_float_tensor(d) self.caches['inputs'][cache_id] = inputs @@ -336,29 +408,35 @@ def _recv_gradients(self, cache_id): if self.grad_tensors is None: if isinstance(outputs, paddle.Tensor): s = list(outputs.shape) - dtype = 'float32' + dtype = 'float16' if self.use_amp else "float32" self.grad_tensors = self._allocate_buffer( s, dtype, num_buffers=1)[0] else: - sizes = [ - list(d.shape) for d in outputs if d.dtype in FLOAT_TYPES - ] - dtypes = ['float32'] * len(sizes) + sizes = [list(d.shape) for d in outputs if is_float_tensor(d)] + dtypes = ['float16'] * len( + sizes) if self.use_amp else ['float32'] * len(sizes) self.grad_tensors = self._allocate_buffers( - sizes, dtypes, num_buffers=1)[0] + sizes, dtypes, num_caches=1)[0] if isinstance(self.grad_tensors, paddle.Tensor): - paddle.distributed.recv(self.grad_tensors, self.next_stage_id) + paddle.distributed.recv( + self.grad_tensors, + self.next_stage_id, + use_calc_stream=True, + group=self.pp_group) else: assert isinstance(outputs, tuple) for d in self.grad_tensors: - paddle.distributed.recv(d, self.next_stage_id) - - def _step(self, lr_kwargs=None): - self._modifying_grad = True + paddle.distributed.recv( + d, + self.next_stage_id, + use_calc_stream=True, + group=self.pp_group) + + def _step(self): + self._allreduce_grads() self.optimizer.step() self.optimizer.clear_gradients() - self._modifying_grad = False def _clear_grads(self, inputs): if isinstance(inputs, paddle.Tensor): @@ -372,26 +450,24 @@ def _clear_grads(self, inputs): def _allocate_zeros(self, shape, dtype): return paddle.zeros(shape, dtype) - def _allocate_buffer(self, shape, dtype, num_buffers=-1, **kwargs): - buffers = [] - if num_buffers == -1: - num_buffers = self.num_caches - for count in range(num_buffers): - buffers.append(self._allocate_zeros(shape, dtype)) - return buffers - - def _allocate_buffers(self, shapes, dtypes, num_buffers=-1): - buffers = [] - if num_buffers == -1: - num_buffers = self.num_caches - for count in range(num_buffers): - buffer = [] + def _allocate_buffer(self, shape, dtype, num_caches=-1): + caches = [] + if num_caches == -1: + num_caches = self.num_caches + for count in range(num_caches): + caches.append(self._allocate_zeros(shape, dtype)) + return caches + + def _allocate_buffers(self, shapes, dtypes, num_caches=-1): + caches = [] + if num_caches == -1: + num_caches = self.num_caches + for count in range(num_caches): + cache = [] for shape, dtype in zip(shapes, dtypes): - buffer.append( - self._allocate_zeros( - shape, dtype, requires_grad=requires_grad)) - buffers.append(buffer) - return buffers + cache.append(self._allocate_zeros(shape, dtype)) + caches.append(cache) + return caches def save_state_dict(self, model_path): state_dict = self._layers.state_dict() @@ -403,25 +479,9 @@ def load_state_dict(self, model_path): _COMMAND_MAP = { utils.Optimize: _step, - #utils.ReduceGrads: _allreduce_grads, utils.Forward: _forward, utils.Backward: _backward, } - def _pre_forward(self, *inputs, **kwargs): - pass - def forward(self, *inputs, **kwargs): raise RuntimeError("Call train_batch for pipeline instead of forward.") - - def _post_forward(self, output): - pass - - def _pre_backward(self, loss): - pass - - def backward_impl(self, loss, parameters): - pass - - def _post_backward(self, loss): - pass diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py index 56eef8d7d21df..7b426e2c3f77d 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py @@ -16,7 +16,21 @@ import paddle from ...utils import hybrid_parallel_util as hp_util -__all__ = ['get_tensor_bytes', ] +__all__ = [ + 'get_tensor_bytes', + 'is_float_tensor', +] + +FLOAT_TYPES = [ + paddle.float16, + paddle.float32, + paddle.float64, +] + + +def is_float_tensor(tensor): + """Is a float tensor""" + return tensor.dtype in FLOAT_TYPES def get_tensor_bytes(tensor): @@ -48,10 +62,6 @@ def __init__(self, micro_batches, stages, stage_id): self.stage_id = stage_id self.prev_stage = self.stage_id - 1 self.next_stage = self.stage_id + 1 - assert self.micro_batches >= self.stages, ( - "micro_batches {} " - "must be greater than or equal to {}".format(self.micro_batches, - self.stages)) @abc.abstractmethod def generate(self): @@ -73,18 +83,25 @@ def generate(self): cmds = [] forward_steps = 0 backward_steps = 0 - while (forward_steps < startup_steps): - cmds.append(Forward) - forward_steps += 1 + #while (forward_steps < startup_steps): + # cmds.append(Forward(cache_id=forward_steps)) + # forward_steps += 1 + #while (forward_steps < self.micro_batches): + # cmds.append(Forward(cache_id=forward_steps)) + # forward_steps += 1 + # cmds.append(Backward(cache_id=backward_steps)) + # backward_steps += 1 + #while (backward_steps < self.micro_batches): + # cmds.append(Backward(cache_id=backward_steps)) + # backward_steps += 1 + #cmds.append(Optimize()) while (forward_steps < self.micro_batches): - cmds.append(Forward) + cmds.append(Forward(cache_id=forward_steps)) forward_steps += 1 - cmds.append(Backward) - backward_steps += 1 while (backward_steps < self.micro_batches): - cmds.append(Backward) + cmds.append(Backward(cache_id=backward_steps)) backward_steps += 1 - cmds.append(Optimize) + cmds.append(Optimize()) yield cmds From d19b5da08e460cccdf952e10d3a2235c8ef41987 Mon Sep 17 00:00:00 2001 From: lilong12 Date: Wed, 5 May 2021 12:54:07 +0800 Subject: [PATCH 028/156] bug fix, test=develop (#32730) --- python/paddle/fluid/optimizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 21b4c429a66e9..41d5401074548 100755 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -4592,13 +4592,13 @@ def _add_sub_blocks(self, main_block, program_list): origin_sub_block_id = op.attr('sub_block').id origin_sub_block = main_program.block(origin_sub_block_id) new_sub_block = prog._create_block(parent_idx=0) - for op in origin_sub_block.ops: - op_desc = op.desc + for sub_op in origin_sub_block.ops: + op_desc = sub_op.desc ap_op = new_sub_block.desc.append_op() ap_op.copy_from(op_desc) new_sub_block._sync_with_cpp() self._create_vars(new_sub_block, origin_sub_block) - op._set_attr('sub_block:', new_sub_block) + op._set_attr('sub_block', new_sub_block) def _get_device_info(self, block): for op in block.ops: From 4626afa44477ff6eb76fa41b72986f53d713e29e Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Wed, 5 May 2021 21:18:50 +0800 Subject: [PATCH 029/156] fix traverse graph in reducer (#32721) --- paddle/fluid/imperative/reducer.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/paddle/fluid/imperative/reducer.cc b/paddle/fluid/imperative/reducer.cc index bf479e0d797ca..e3dd0a2aa75b4 100644 --- a/paddle/fluid/imperative/reducer.cc +++ b/paddle/fluid/imperative/reducer.cc @@ -443,10 +443,6 @@ void Reducer::PrepareDeps(const std::unordered_set &init_nodes) { auto *cur_node = q.front(); q.pop(); - for (auto &cur_op : *cur_node) { - cur_op.EnforceHasInOut(); - } - const auto &grad_pending_nodes = cur_node->GradPendingNodes(); for (auto &grad_pending_node : grad_pending_nodes) { PADDLE_ENFORCE_NOT_NULL( @@ -523,7 +519,6 @@ void Reducer::PrepareForBackward( q.pop(); for (const auto &cur_op : *cur_node) { - cur_op.EnforceHasInOut(); auto &bwd_outs = cur_op.GetOutsMap(); for (const auto &pair : bwd_outs) { if (!pair.second.IsGrad()) { From cdfc34d278a784fa856487db9f2434b93836ff05 Mon Sep 17 00:00:00 2001 From: Huihuang Zheng Date: Thu, 6 May 2021 10:32:47 +0800 Subject: [PATCH 030/156] [Dy2stat] Fix to_tensor Bug Reported from QA (#32701) (#32713) Dy2stat failed when user writes return paddle.to_tensor(xxx), the reason is that visit_Expr doesn't work when the Expr is in return. Some other statements may trigger same bug. To fix it, we re-wrote a transformer to transform paddle.to_tensor to paddle.assign for all Call nodes. --- .../basic_api_transformer.py | 33 +++++++++++++++---- .../test_basic_api_transformation.py | 6 ++-- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py index 198c2920eec7f..5ea1fdfac0928 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py @@ -33,10 +33,11 @@ def __init__(self, wrapper_root): self.root = wrapper_root.node self.class_node_dict = {} - self.name_to_tensor_shape = {} - def transform(self): + to_tensor_transformer = ToTensorTransformer(self.root) + to_tensor_transformer.transform() self.visit(self.root) + return self.wrapper_root def visit_Assign(self, node): @@ -62,11 +63,6 @@ def visit_Expr(self, node): def _visit_Call(self, node): assert isinstance(node, gast.Call) - # Replace API `to_variable` with `fluid.layers.assign` - if is_to_variable(node): - node = to_assign_node(node) - return node - func_name = astor.to_source(gast.gast_to_ast(node.func)) if self._is_dygraph_forward(func_name): @@ -102,6 +98,29 @@ def _update_class_node_dict(self, node): return False +class ToTensorTransformer(gast.NodeTransformer): + """ + Class to transform paddle.to_tensor and paddle.to_variable to paddle.assign + """ + + def __init__(self, node): + assert isinstance( + node, gast.AST + ), "Input non-gast.AST node for the initialization of ToTensorTransformer." + self.root = node + + def transform(self): + self.visit(self.root) + return self.root + + def visit_Call(self, node): + assert isinstance(node, gast.Call) + if is_to_variable(node): + node = to_assign_node(node) + self.generic_visit(node) + return node + + def is_to_variable(node): assert isinstance(node, gast.Call) api_name = utils.ast_to_source_code(node.func).strip() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py index 630b804f9a2fb..ea745ad661425 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py @@ -64,13 +64,11 @@ def dyfunc_int_to_tensor(x): def dyfunc_float_to_tensor(x): - res = paddle.to_tensor(2.0) - return res + return paddle.to_tensor(2.0) def dyfunc_bool_to_tensor(x): - res = paddle.to_tensor(True) - return res + return paddle.to_tensor(True) class TestDygraphBasicApi_ToVariable(unittest.TestCase): From 035c74252286bdae2dba501da07dbe2ff0d1addf Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Thu, 6 May 2021 10:40:08 +0800 Subject: [PATCH 031/156] add API Tensor.item() to convert Tensor element to a Python scalar (#32634) cherry-pick #32561 --- paddle/fluid/pybind/imperative.cc | 64 +++++++++++++++++ .../fluid/dygraph/varbase_patch_methods.py | 70 ++++++++++++++++++- .../fluid/tests/unittests/test_var_base.py | 68 ++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index 93441eb52fe5e..450c992d41118 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -784,6 +784,70 @@ void BindImperative(py::module *m_ptr) { return out; } }) + .def( + "_getitem_from_offset", + [](std::shared_ptr &self, const py::args &args) { + const auto &tensor = self->Var().Get(); + PADDLE_ENFORCE_EQ( + tensor.IsInitialized(), true, + platform::errors::InvalidArgument( + "Tensor of %s is Empty, please check if it has no data.", + self->Name())); + + const auto &tensor_dims = tensor.dims(); + + std::vector dims(tensor_dims.size()); + std::vector strides(tensor_dims.size()); + + size_t numel = 1; + for (int i = tensor_dims.size() - 1; i >= 0; --i) { + strides[i] = numel; + dims[i] = static_cast(tensor_dims[i]); + numel *= dims[i]; + } + size_t offset = 0; + if (args.empty()) { + PADDLE_ENFORCE_EQ( + numel, 1, + platform::errors::InvalidArgument( + "only one element tensors can be converted to Python " + "scalars when no input coordinates")); + } else if (args.size() == 1) { + offset = args[0].cast(); + PADDLE_ENFORCE_LT( + offset, numel, + platform::errors::InvalidArgument( + "index %d is out of bounds for size %d", offset, numel)); + } else { + PADDLE_ENFORCE_EQ(args.size(), dims.size(), + platform::errors::InvalidArgument( + "incorrect number of indices for Tensor")); + + for (size_t i = 0; i < args.size(); ++i) { + size_t index = args[i].cast(); + PADDLE_ENFORCE_LT( + index, dims[i], + platform::errors::InvalidArgument( + "index %d is out fo bounds for axis %d with size %d", + index, i, dims[i])); + offset += index * strides[i]; + } + } +#define TENSOR_TO_PY_SCALAR(T, proto_type) \ + if (tensor.type() == proto_type) { \ + std::string py_dtype_str = details::TensorDTypeToPyDTypeStr(proto_type); \ + T b = TensorGetElement(tensor, offset); \ + return py::array(py::dtype(py_dtype_str.c_str()), {}, {}, \ + static_cast(&b)); \ + } + + _ForEachDataType_(TENSOR_TO_PY_SCALAR); +#undef TENSOR_TO_PY_SCALAR + PADDLE_THROW(platform::errors::Unimplemented( + "Unsupported tensor data type: %s", + framework::DataTypeToString(tensor.type()))); + }, + py::return_value_policy::copy) .def("_inplace_version", [](imperative::VarBase &self) -> uint32_t { const auto &var = self.MutableVar(); diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index dbc2b24aeeaae..bb84b2ca9705c 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -375,6 +375,49 @@ def clear_grad(self): """ self.clear_gradient() + def item(self, *args): + """ + Convert one element Tensor to a Python scalar. + + Args: + *args(int): The input coordinates. If it's single int, the data in the corresponding order of flattened Tensor will be returned. + Default: None, and it must be in the case where Tensor has only one element. + + Returns(Python scalar): A Python scalar, whose dtype is corresponds to the dtype of Tensor. + + Raises: + ValueError: If the Tensor has more than one element, there must be coordinates. + + Examples: + .. code-block:: python + + import paddle + + x = paddle.to_tensor(1) + print(x.item()) #1 + print(type(x.item())) # + + x = paddle.to_tensor(1.0) + print(x.item()) #1.0 + print(type(x.item())) # + + x = paddle.to_tensor(True) + print(x.item()) #True + print(type(x.item())) # + + x = paddle.to_tensor(1+1j) + print(x.item()) #(1+1j) + print(type(x.item())) # + + x = paddle.to_tensor([[1.1, 2.2, 3.3]]) + print(x.item(2)) #3.3 + print(x.item(0, 2)) #3.3 + + x = paddle.to_tensor([1, 2]) + x.item() #ValueError: only one element tensor can be converted to Python scalar when no input coordinates. + """ + return self._getitem_from_offset(*args).item() + @property def inplace_version(self): """ @@ -462,7 +505,30 @@ def __bool__(self): return self.__nonzero__() def __array__(self, dtype=None): - return self.numpy().astype(dtype) + """ + Returns a numpy array shows the value of current Tensor. + + Returns: + ndarray: The numpy value of current Tensor. + + Returns type: + ndarray: dtype is same as current Tensor + + Examples: + .. code-block:: python + + import paddle + import numpy as np + x = paddle.randn([2, 2]) + x_array = np.array(x) + + print(type(x_array)) # + print(x_array.shape) #(2, 2) + """ + array = self.numpy() + if dtype: + array = array.astype(dtype) + return array def __getitem__(self, item): def contain_tensor(item): @@ -498,7 +564,7 @@ def contain_tensor(item): ("__str__", __str__), ("__repr__", __str__), ("__deepcopy__", __deepcopy__), ("__module__", "paddle"), ("__name__", "Tensor"), ("__array__", __array__), - ("__getitem__", __getitem__)): + ("__getitem__", __getitem__), ("item", item)): setattr(core.VarBase, method_name, method) # NOTE(zhiqiu): pybind11 will set a default __str__ method of enum class. diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py index a65308c84e719..0b9159af00869 100644 --- a/python/paddle/fluid/tests/unittests/test_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_var_base.py @@ -143,6 +143,74 @@ def _test_place(place): self.assertEqual(y.dtype, core.VarDesc.VarType.COMPLEX64) self.assertEqual(y.shape, [2]) + paddle.set_default_dtype('float32') + x = paddle.randn([3, 4]) + x_array = np.array(x) + self.assertEqual(x_array.shape, x.numpy().shape) + self.assertEqual(x_array.dtype, x.numpy().dtype) + self.assertTrue(np.array_equal(x_array, x.numpy())) + + x = paddle.to_tensor(1.0) + self.assertEqual(x.item(), 1.0) + self.assertTrue(isinstance(x.item(), float)) + + x = paddle.randn([3, 2, 2]) + self.assertTrue(isinstance(x.item(5), float)) + self.assertTrue(isinstance(x.item(1, 0, 1), float)) + self.assertEqual(x.item(5), x.item(1, 0, 1)) + self.assertTrue( + np.array_equal(x.item(1, 0, 1), x.numpy().item(1, 0, 1))) + + x = paddle.to_tensor([[1.111111, 2.222222, 3.333333]]) + self.assertEqual(x.item(0, 2), x.item(2)) + self.assertAlmostEqual(x.item(2), 3.333333) + self.assertTrue(isinstance(x.item(0, 2), float)) + + x = paddle.to_tensor(1.0, dtype='float64') + self.assertEqual(x.item(), 1.0) + self.assertTrue(isinstance(x.item(), float)) + + x = paddle.to_tensor(1.0, dtype='float16') + self.assertEqual(x.item(), 1.0) + self.assertTrue(isinstance(x.item(), float)) + + x = paddle.to_tensor(1, dtype='uint8') + self.assertEqual(x.item(), 1) + print(type(x.item())) + self.assertTrue(isinstance(x.item(), int)) + + x = paddle.to_tensor(1, dtype='int8') + self.assertEqual(x.item(), 1) + self.assertTrue(isinstance(x.item(), int)) + + x = paddle.to_tensor(1, dtype='int16') + self.assertEqual(x.item(), 1) + self.assertTrue(isinstance(x.item(), int)) + + x = paddle.to_tensor(1, dtype='int32') + self.assertEqual(x.item(), 1) + self.assertTrue(isinstance(x.item(), int)) + + x = paddle.to_tensor(1, dtype='int64') + self.assertEqual(x.item(), 1) + self.assertTrue(isinstance(x.item(), long if six.PY2 else int)) + + x = paddle.to_tensor(True) + self.assertEqual(x.item(), True) + self.assertTrue(isinstance(x.item(), bool)) + + x = paddle.to_tensor(1 + 1j) + self.assertEqual(x.item(), 1 + 1j) + self.assertTrue(isinstance(x.item(), complex)) + + with self.assertRaises(ValueError): + paddle.randn([3, 2, 2]).item() + with self.assertRaises(ValueError): + paddle.randn([3, 2, 2]).item(18) + with self.assertRaises(ValueError): + paddle.randn([3, 2, 2]).item(1, 2) + with self.assertRaises(ValueError): + paddle.randn([3, 2, 2]).item(2, 1, 2) with self.assertRaises(TypeError): paddle.to_tensor('test') with self.assertRaises(TypeError): From df00636bad51595793dcc3b59073dca72480cb37 Mon Sep 17 00:00:00 2001 From: lilong12 Date: Thu, 6 May 2021 11:35:55 +0800 Subject: [PATCH 032/156] update, test=develop (#32731) --- paddle/fluid/pybind/op_function_generator.cc | 1 - python/paddle/distributed/collective.py | 46 ++++++++++---------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/pybind/op_function_generator.cc b/paddle/fluid/pybind/op_function_generator.cc index a340d7a0f00d9..bf3c77843219c 100644 --- a/paddle/fluid/pybind/op_function_generator.cc +++ b/paddle/fluid/pybind/op_function_generator.cc @@ -44,7 +44,6 @@ std::map> op_ins_map = { {"gru_unit", {"Input", "HiddenPrev", "Weight", "Bias"}}, {"label_smooth", {"X", "PriorDist"}}, {"assign", {"X"}}, - {"send_v2", {"X"}}, {"reshape2", {"X", "Shape"}}, {"expand", {"X", "ExpandTimes"}}, {"slice", {"Input", "StartsTensor", "EndsTensor"}}, diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index 69a8f8956a8c1..fefabaf69768e 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -1186,23 +1186,24 @@ def send(tensor, dst=0, group=None, use_calc_stream=True): tensor (Tensor): The Tensor to send. Its data type should be float16, float32, float64, int32 or int64. dst (int): The destination rank id. - group (Group): The group instance return by new_group or None for global default group. - use_calc_stream (bool): Whether to use calculate stream or communication stream. + group (Group, optional): The group instance return by new_group or None for global default group. Default: None. + use_calc_stream (bool, optional): Whether to use calculate stream or communication stream. Default: True. Returns: None. Examples: .. code-block:: python + # required: distributed import paddle - #from paddle.distributed import init_parallel_env - #init_parallel_env() - #if paddle.distributed.ParallelEnv().rank == 0: - # data = paddle.to_tensor([7, 8, 9]) - # paddle.distributed.send(data, dst=1) - #else: - # data = paddle.to_tensor([1,2,3]) - # paddle.distributed.recv(data, src=0) - #out = data.numpy() + from paddle.distributed import init_parallel_env + init_parallel_env() + if paddle.distributed.ParallelEnv().rank == 0: + data = paddle.to_tensor([7, 8, 9]) + paddle.distributed.send(data, dst=1) + else: + data = paddle.to_tensor([1,2,3]) + paddle.distributed.recv(data, src=0) + out = data.numpy() """ if group is not None and not group.is_member(): return @@ -1235,23 +1236,24 @@ def recv(tensor, src=0, group=None, use_calc_stream=True): tensor (Tensor): The Tensor to receive. Its data type should be float16, float32, float64, int32 or int64. src (int): The source rank id. - group (Group): The group instance return by new_group or None for global default group. - use_calc_stream (bool): Whether to use calculate stream or communication stream. + group (Group, optional): The group instance return by new_group or None for global default group. Default: None. + use_calc_stream (bool, optional): Whether to use calculate stream or communication stream. Default: True. Returns: None. Examples: .. code-block:: python + # required: distributed import paddle - #from paddle.distributed import init_parallel_env - #init_parallel_env() - #if paddle.distributed.ParallelEnv().rank == 0: - # data = paddle.to_tensor([7, 8, 9]) - # paddle.distributed.send(data, dst=1) - #else: - # data = paddle.to_tensor([1,2,3]) - # paddle.distributed.recv(data, src=0) - #out = data.numpy() + from paddle.distributed import init_parallel_env + init_parallel_env() + if paddle.distributed.ParallelEnv().rank == 0: + data = paddle.to_tensor([7, 8, 9]) + paddle.distributed.send(data, dst=1) + else: + data = paddle.to_tensor([1,2,3]) + paddle.distributed.recv(data, src=0) + out = data.numpy() """ if group is not None and not group.is_member(): return From c0f266835b8e693e8b0c7c3f593980aed9b6149f Mon Sep 17 00:00:00 2001 From: littletomatodonkey <2120160898@bit.edu.cn> Date: Thu, 6 May 2021 11:44:44 +0800 Subject: [PATCH 033/156] fix l1 decay for inplace (#32718) --- python/paddle/fluid/regularizer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index db08955c455fb..64ce283a63c5b 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -326,19 +326,21 @@ def __call__(self, param, grad, block): assert isinstance(block, framework.Block) if framework.in_dygraph_mode(): + sign = block.create_var(dtype=param.dtype, shape=param.shape) decay = block.create_var(dtype=param.dtype, shape=param.shape) else: + sign = block.create_var( + dtype=param.dtype, shape=param.shape, lod_level=param.lod_level) decay = block.create_var( dtype=param.dtype, shape=param.shape, lod_level=param.lod_level) # Append sign op - block.append_op( - type='sign', inputs={"X": param}, outputs={"Out": decay}) + block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign}) # Append scale op to the output of sign op block.append_op( type='scale', - inputs={"X": decay}, + inputs={"X": sign}, outputs={"Out": decay}, attrs={"scale": self._regularization_coeff}) From 43b3e99f53c45d17622cbb434f2e66a315516cf7 Mon Sep 17 00:00:00 2001 From: Wenyu Date: Thu, 6 May 2021 15:13:57 +0800 Subject: [PATCH 034/156] fix error imformation when trigger import error (#32702) --- python/paddle/hapi/hub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/hapi/hub.py b/python/paddle/hapi/hub.py index 6490c878f9b88..54765c1d4d41c 100644 --- a/python/paddle/hapi/hub.py +++ b/python/paddle/hapi/hub.py @@ -43,8 +43,8 @@ def _import_module(name, repo_dir): except ImportError: sys.path.remove(repo_dir) raise RuntimeError( - 'Cannot import `{}`, please make sure `{}`.py in repo root dir'. - format(name, name)) + 'Please make sure config exists or repo error messages above fixed when importing' + ) sys.path.remove(repo_dir) From a9d330a390b7151fdc363413537afc30b8d3bcd1 Mon Sep 17 00:00:00 2001 From: Wenyu Date: Thu, 6 May 2021 15:46:29 +0800 Subject: [PATCH 035/156] [cherry-pick pr31970] Support transforms for paddle tensor image (#32705) * add to_grayscale, normalize * add rotate * add vfip and hflip * add crop center_crop * add utils * add utils * update utils, add raise for some cases * add padding, support constant, reflect, replicate, circular same as paddle.pad * update rotate * using utils func in [v|h]flip * add get-image-[n,c,w,h] axis utils * add get-image-[n,c,w,h] axis utils * align * update * remove default value in utils func * add assert for pad * update assert paddle image * support rotate fill func * raise valueerror for pad * remove typing, py2 dont support * init uinttest for transforms tensor * add resize op * register [normalize hflip crop center_crop resize transpose] imagenet * register [normalize hflip crop center_crop resize transpose] imagenet * fix bugs, (w, h) getter and import * add _get_image_size for tensor image * add pad vflip for tensor image * add unittest for tensor transforms * update transforms unittest for converage CI probelms, test=develop * update * update * update * fix `get_shape` for tensor backend * update * update * add more resize tests * update * update for ci test * update * remove redundancy code * update uinttest, and set tensor image to hwc by default * add tensor backend * fix copyright doc, rm comment code, add pil unittest * update data_format to `chw` for tensor * coverage notest,test=coverage * update * update --- python/paddle/tests/test_transforms.py | 230 ++++++++- python/paddle/vision/image.py | 10 +- python/paddle/vision/transforms/functional.py | 75 ++- .../vision/transforms/functional_tensor.py | 488 +++++++++++++++++- python/paddle/vision/transforms/transforms.py | 5 + 5 files changed, 764 insertions(+), 44 deletions(-) diff --git a/python/paddle/tests/test_transforms.py b/python/paddle/tests/test_transforms.py index 5086a12d945bc..c84950fdbc539 100644 --- a/python/paddle/tests/test_transforms.py +++ b/python/paddle/tests/test_transforms.py @@ -56,7 +56,10 @@ def create_image(self, shape): 'uint8')) def get_shape(self, img): - if self.backend == 'pil': + if isinstance(img, paddle.Tensor): + return img.shape + + elif self.backend == 'pil': return np.array(img).shape return img.shape @@ -253,6 +256,22 @@ def test_exception(self): fake_img = self.create_image((100, 120, 3)) F.pad(fake_img, [1.0, 2.0, 3.0]) + with self.assertRaises(TypeError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, '1') + + with self.assertRaises(TypeError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, 1, {}) + + with self.assertRaises(TypeError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, 1, padding_mode=-1) + + with self.assertRaises(ValueError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, [1.0, 2.0, 3.0]) + with self.assertRaises(ValueError): transforms.RandomRotation(-2) @@ -290,6 +309,159 @@ def get_backend(self): return 'pil' +class TestTransformsTensor(TestTransformsCV2): + def get_backend(self): + return 'tensor' + + def create_image(self, shape): + return paddle.to_tensor(np.random.rand(*shape)).transpose( + (2, 0, 1)) # hwc->chw + + def do_transform(self, trans): + trans.transforms.insert(0, transforms.ToTensor(data_format='CHW')) + trans.transforms.append(transforms.Transpose(order=(1, 2, 0))) + dataset_folder = DatasetFolder(self.data_dir, transform=trans) + for _ in dataset_folder: + pass + + def test_trans_all(self): + normalize = transforms.Normalize( + mean=[123.675, 116.28, 103.53], + std=[58.395, 57.120, 57.375], ) + trans = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + normalize, + ]) + self.do_transform(trans) + + def test_grayscale(self): + trans = transforms.Compose([transforms.Grayscale()]) + self.do_transform(trans) + + trans_gray = transforms.Grayscale() + fake_img = self.create_image((500, 400, 3)) + fake_img_gray = trans_gray(fake_img) + + np.testing.assert_equal(self.get_shape(fake_img_gray)[1], 500) + np.testing.assert_equal(self.get_shape(fake_img_gray)[2], 400) + + trans_gray3 = transforms.Grayscale(3) + fake_img = self.create_image((500, 400, 3)) + fake_img_gray = trans_gray3(fake_img) + + def test_normalize(self): + normalize = transforms.Normalize(mean=0.5, std=0.5) + trans = transforms.Compose([normalize]) + self.do_transform(trans) + + def test_pad(self): + trans = transforms.Compose([transforms.Pad(2)]) + self.do_transform(trans) + + fake_img = self.create_image((200, 150, 3)) + trans_pad = transforms.Compose([transforms.Pad(10)]) + fake_img_padded = trans_pad(fake_img) + np.testing.assert_equal(self.get_shape(fake_img_padded), (3, 220, 170)) + trans_pad1 = transforms.Pad([1, 2]) + trans_pad2 = transforms.Pad([1, 2, 3, 4]) + trans_pad4 = transforms.Pad(1, padding_mode='edge') + img = trans_pad1(fake_img) + img = trans_pad2(img) + img = trans_pad4(img) + + def test_random_crop(self): + trans = transforms.Compose([ + transforms.RandomCrop(200), + transforms.RandomCrop((140, 160)), + ]) + self.do_transform(trans) + + trans_random_crop1 = transforms.RandomCrop(224) + trans_random_crop2 = transforms.RandomCrop((140, 160)) + + fake_img = self.create_image((500, 400, 3)) + fake_img_crop1 = trans_random_crop1(fake_img) + fake_img_crop2 = trans_random_crop2(fake_img_crop1) + + np.testing.assert_equal(self.get_shape(fake_img_crop1), (3, 224, 224)) + + np.testing.assert_equal(self.get_shape(fake_img_crop2), (3, 140, 160)) + + trans_random_crop_same = transforms.RandomCrop((140, 160)) + img = trans_random_crop_same(fake_img_crop2) + + trans_random_crop_bigger = transforms.RandomCrop( + (180, 200), pad_if_needed=True) + img = trans_random_crop_bigger(img) + + trans_random_crop_pad = transforms.RandomCrop((224, 256), 2, True) + img = trans_random_crop_pad(img) + + def test_exception(self): + trans = transforms.Compose([transforms.Resize(-1)]) + + trans_batch = transforms.Compose([transforms.Resize(-1)]) + + with self.assertRaises(Exception): + self.do_transform(trans) + + with self.assertRaises(Exception): + self.do_transform(trans_batch) + + with self.assertRaises(ValueError): + transforms.Pad([1.0, 2.0, 3.0]) + + with self.assertRaises(TypeError): + fake_img = self.create_image((100, 120, 3)) + F.pad(fake_img, '1') + + with self.assertRaises(TypeError): + fake_img = self.create_image((100, 120, 3)) + F.pad(fake_img, 1, {}) + + with self.assertRaises(TypeError): + fake_img = self.create_image((100, 120, 3)) + F.pad(fake_img, 1, padding_mode=-1) + + with self.assertRaises(ValueError): + fake_img = self.create_image((100, 120, 3)) + F.pad(fake_img, [1.0, 2.0, 3.0]) + + with self.assertRaises(TypeError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, '1') + + with self.assertRaises(TypeError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, 1, {}) + + with self.assertRaises(TypeError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, 1, padding_mode=-1) + + with self.assertRaises(ValueError): + tensor_img = paddle.rand((3, 100, 100)) + F.pad(tensor_img, [1.0, 2.0, 3.0]) + + with self.assertRaises(ValueError): + transforms.RandomRotation(-2) + + with self.assertRaises(ValueError): + transforms.RandomRotation([1, 2, 3]) + + with self.assertRaises(ValueError): + trans_gray = transforms.Grayscale(5) + fake_img = self.create_image((100, 120, 3)) + trans_gray(fake_img) + + with self.assertRaises(TypeError): + transform = transforms.RandomResizedCrop(64) + transform(1) + + test_color_jitter = None + + class TestFunctional(unittest.TestCase): def test_errors(self): with self.assertRaises(TypeError): @@ -300,6 +472,14 @@ def test_errors(self): 'uint8')) F.to_tensor(fake_img, data_format=1) + with self.assertRaises(ValueError): + fake_img = paddle.rand((3, 100, 100)) + F.pad(fake_img, 1, padding_mode='symmetric') + + with self.assertRaises(TypeError): + fake_img = paddle.rand((3, 100, 100)) + F.resize(fake_img, {1: 1}) + with self.assertRaises(TypeError): fake_img = Image.fromarray((np.random.rand(28, 28, 3) * 255).astype( 'uint8')) @@ -354,31 +534,50 @@ def test_normalize(self): std = [0.5, 0.5, 0.5] normalized_img = F.normalize(tensor_img, mean, std) - normalized_img = F.normalize( + normalized_img_tensor = F.normalize( tensor_img_hwc, mean, std, data_format='HWC') - normalized_img = F.normalize(pil_img, mean, std, data_format='HWC') - normalized_img = F.normalize( + normalized_img_pil = F.normalize(pil_img, mean, std, data_format='HWC') + normalized_img_np = F.normalize( np_img, mean, std, data_format='HWC', to_rgb=True) + np.testing.assert_almost_equal( + np.array(normalized_img_pil), normalized_img_np) + np.testing.assert_almost_equal(normalized_img_tensor.numpy(), + normalized_img_np) + def test_center_crop(self): np_img = (np.random.rand(28, 24, 3)).astype('uint8') pil_img = Image.fromarray(np_img) + tensor_img = F.to_tensor(pil_img, data_format='CHW') np_cropped_img = F.center_crop(np_img, 4) pil_cropped_img = F.center_crop(pil_img, 4) + tensor_cropped_img = F.center_crop(tensor_img, 4) np.testing.assert_almost_equal(np_cropped_img, np.array(pil_cropped_img)) + np.testing.assert_almost_equal(np_cropped_img, + tensor_cropped_img.numpy().transpose( + (1, 2, 0))) def test_pad(self): np_img = (np.random.rand(28, 24, 3)).astype('uint8') pil_img = Image.fromarray(np_img) + tensor_img = F.to_tensor(pil_img, 'CHW') np_padded_img = F.pad(np_img, [1, 2], padding_mode='reflect') pil_padded_img = F.pad(pil_img, [1, 2], padding_mode='reflect') + tensor_padded_img = F.pad(tensor_img, [1, 2], padding_mode='reflect') np.testing.assert_almost_equal(np_padded_img, np.array(pil_padded_img)) + np.testing.assert_almost_equal(np_padded_img, + tensor_padded_img.numpy().transpose( + (1, 2, 0))) + + tensor_padded_img = F.pad(tensor_img, 1, padding_mode='reflect') + tensor_padded_img = F.pad(tensor_img, [1, 2, 1, 2], + padding_mode='reflect') pil_p_img = pil_img.convert('P') pil_padded_img = F.pad(pil_p_img, [1, 2]) @@ -387,12 +586,21 @@ def test_pad(self): def test_resize(self): np_img = (np.zeros([28, 24, 3])).astype('uint8') pil_img = Image.fromarray(np_img) + tensor_img = F.to_tensor(pil_img, 'CHW') np_reseized_img = F.resize(np_img, 40) pil_reseized_img = F.resize(pil_img, 40) + tensor_reseized_img = F.resize(tensor_img, 40) + tensor_reseized_img2 = F.resize(tensor_img, (46, 40)) np.testing.assert_almost_equal(np_reseized_img, np.array(pil_reseized_img)) + np.testing.assert_almost_equal(np_reseized_img, + tensor_reseized_img.numpy().transpose( + (1, 2, 0))) + np.testing.assert_almost_equal(np_reseized_img, + tensor_reseized_img2.numpy().transpose( + (1, 2, 0))) gray_img = (np.zeros([28, 32])).astype('uint8') gray_resize_img = F.resize(gray_img, 40) @@ -447,12 +655,24 @@ def test_image_load(self): def test_rotate(self): np_img = (np.random.rand(28, 28, 3) * 255).astype('uint8') pil_img = Image.fromarray(np_img).convert('RGB') - rotated_np_img = F.rotate(np_img, 80, expand=True) rotated_pil_img = F.rotate(pil_img, 80, expand=True) + tensor_img = F.to_tensor(pil_img, 'CHW') + + rotated_tensor_img1 = F.rotate(tensor_img, 80, expand=True) + + rotated_tensor_img2 = F.rotate( + tensor_img, + 80, + interpolation='bilinear', + center=(10, 10), + expand=False) + np.testing.assert_equal(rotated_np_img.shape, np.array(rotated_pil_img).shape) + np.testing.assert_equal(rotated_np_img.shape, + rotated_tensor_img1.transpose((1, 2, 0)).shape) def test_rotate1(self): np_img = (np.random.rand(28, 28, 3) * 255).astype('uint8') diff --git a/python/paddle/vision/image.py b/python/paddle/vision/image.py index 3d5ea3a73af6c..19986816b7cc4 100644 --- a/python/paddle/vision/image.py +++ b/python/paddle/vision/image.py @@ -80,9 +80,9 @@ def make_fake_dir(): shutil.rmtree(temp_dir) """ global _image_backend - if backend not in ['pil', 'cv2']: + if backend not in ['pil', 'cv2', 'tensor']: raise ValueError( - "Expected backend are one of ['pil', 'cv2'], but got {}" + "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}" .format(backend)) _image_backend = backend @@ -150,13 +150,13 @@ def image_load(path, backend=None): if backend is None: backend = _image_backend - if backend not in ['pil', 'cv2']: + if backend not in ['pil', 'cv2', 'tensor']: raise ValueError( - "Expected backend are one of ['pil', 'cv2'], but got {}" + "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}" .format(backend)) if backend == 'pil': return Image.open(path) - else: + elif backend == 'cv2': cv2 = try_import('cv2') return cv2.imread(path) diff --git a/python/paddle/vision/transforms/functional.py b/python/paddle/vision/transforms/functional.py index c0e72877ffcdd..18a35915c99da 100644 --- a/python/paddle/vision/transforms/functional.py +++ b/python/paddle/vision/transforms/functional.py @@ -25,13 +25,6 @@ from numpy import sin, cos, tan import paddle -if sys.version_info < (3, 3): - Sequence = collections.Sequence - Iterable = collections.Iterable -else: - Sequence = collections.abc.Sequence - Iterable = collections.abc.Iterable - from . import functional_pil as F_pil from . import functional_cv2 as F_cv2 from . import functional_tensor as F_t @@ -83,14 +76,18 @@ def to_tensor(pic, data_format='CHW'): print(tensor.shape) """ - if not (_is_pil_image(pic) or _is_numpy_image(pic)): - raise TypeError('pic should be PIL Image or ndarray. Got {}'.format( - type(pic))) + if not (_is_pil_image(pic) or _is_numpy_image(pic) or + _is_tensor_image(pic)): + raise TypeError( + 'pic should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. + format(type(pic))) if _is_pil_image(pic): return F_pil.to_tensor(pic, data_format) - else: + elif _is_numpy_image(pic): return F_cv2.to_tensor(pic, data_format) + else: + return pic if data_format.lower() == 'chw' else pic.transpose((1, 2, 0)) def resize(img, size, interpolation='bilinear'): @@ -135,13 +132,16 @@ def resize(img, size, interpolation='bilinear'): converted_img = F.resize(fake_img, (200, 150)) print(converted_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if _is_pil_image(img): return F_pil.resize(img, size, interpolation) + elif _is_tensor_image(img): + return F_t.resize(img, size, interpolation) else: return F_cv2.resize(img, size, interpolation) @@ -196,13 +196,16 @@ def pad(img, padding, fill=0, padding_mode='constant'): padded_img = F.pad(fake_img, padding=(2, 1)) print(padded_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if _is_pil_image(img): return F_pil.pad(img, padding, fill, padding_mode) + elif _is_tensor_image(img): + return F_t.pad(img, padding, fill, padding_mode) else: return F_cv2.pad(img, padding, fill, padding_mode) @@ -236,13 +239,16 @@ def crop(img, top, left, height, width): print(cropped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if _is_pil_image(img): return F_pil.crop(img, top, left, height, width) + elif _is_tensor_image(img): + return F_t.crop(img, top, left, height, width) else: return F_cv2.crop(img, top, left, height, width) @@ -272,13 +278,16 @@ def center_crop(img, output_size): cropped_img = F.center_crop(fake_img, (150, 100)) print(cropped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if _is_pil_image(img): return F_pil.center_crop(img, output_size) + elif _is_tensor_image(img): + return F_t.center_crop(img, output_size) else: return F_cv2.center_crop(img, output_size) @@ -307,13 +316,16 @@ def hflip(img): print(flpped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if _is_pil_image(img): return F_pil.hflip(img) + elif _is_tensor_image(img): + return F_t.hflip(img) else: return F_cv2.hflip(img) @@ -342,13 +354,16 @@ def vflip(img): print(flpped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if _is_pil_image(img): return F_pil.vflip(img) + elif _is_tensor_image(img): + return F_t.vflip(img) else: return F_cv2.vflip(img) @@ -563,9 +578,10 @@ def rotate(img, print(rotated_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if isinstance(center, list): @@ -575,6 +591,8 @@ def rotate(img, if _is_pil_image(img): return F_pil.rotate(img, angle, interpolation, expand, center, fill) + elif _is_tensor_image(img): + return F_t.rotate(img, angle, interpolation, expand, center, fill) else: return F_cv2.rotate(img, angle, interpolation, expand, center, fill) @@ -606,13 +624,16 @@ def to_grayscale(img, num_output_channels=1): print(gray_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img)): + if not (_is_pil_image(img) or _is_numpy_image(img) or + _is_tensor_image(img)): raise TypeError( - 'img should be PIL Image or ndarray with dim=[2 or 3]. Got {}'. + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'. format(type(img))) if _is_pil_image(img): return F_pil.to_grayscale(img, num_output_channels) + elif _is_tensor_image(img): + return F_t.to_grayscale(img, num_output_channels) else: return F_cv2.to_grayscale(img, num_output_channels) diff --git a/python/paddle/vision/transforms/functional_tensor.py b/python/paddle/vision/transforms/functional_tensor.py index e8b70820dd9af..7f490d57916fb 100644 --- a/python/paddle/vision/transforms/functional_tensor.py +++ b/python/paddle/vision/transforms/functional_tensor.py @@ -14,11 +14,78 @@ from __future__ import division +import math +import numbers + import paddle +import paddle.nn.functional as F + +import sys +import collections + + +def _assert_image_tensor(img, data_format): + if not isinstance( + img, paddle.Tensor) or img.ndim != 3 or not data_format.lower() in ( + 'chw', 'hwc'): + raise RuntimeError( + 'not support [type={}, ndim={}, data_format={}] paddle image'. + format(type(img), img.ndim, data_format)) + + +def _get_image_h_axis(data_format): + if data_format.lower() == 'chw': + return -2 + elif data_format.lower() == 'hwc': + return -3 + + +def _get_image_w_axis(data_format): + if data_format.lower() == 'chw': + return -1 + elif data_format.lower() == 'hwc': + return -2 + + +def _get_image_c_axis(data_format): + if data_format.lower() == 'chw': + return -3 + elif data_format.lower() == 'hwc': + return -1 + + +def _get_image_n_axis(data_format): + if len(data_format) == 3: + return None + elif len(data_format) == 4: + return 0 + + +def _is_channel_last(data_format): + return _get_image_c_axis(data_format) == -1 + + +def _is_channel_first(data_format): + return _get_image_c_axis(data_format) == -3 + + +def _get_image_num_batches(img, data_format): + if _get_image_n_axis(data_format): + return img.shape[_get_image_n_axis(data_format)] + return None + + +def _get_image_num_channels(img, data_format): + return img.shape[_get_image_c_axis(data_format)] + + +def _get_image_size(img, data_format): + return img.shape[_get_image_w_axis(data_format)], img.shape[ + _get_image_h_axis(data_format)] def normalize(img, mean, std, data_format='CHW'): - """Normalizes a tensor image with mean and standard deviation. + """Normalizes a tensor image given mean and standard deviation. Args: img (paddle.Tensor): input data to be normalized. @@ -31,10 +98,417 @@ def normalize(img, mean, std, data_format='CHW'): Tensor: Normalized mage. """ - if data_format == 'CHW': - mean = paddle.to_tensor(mean).reshape([-1, 1, 1]) - std = paddle.to_tensor(std).reshape([-1, 1, 1]) - else: - mean = paddle.to_tensor(mean) - std = paddle.to_tensor(std) + _assert_image_tensor(img, data_format) + + mean = paddle.to_tensor(mean, place=img.place) + std = paddle.to_tensor(std, place=img.place) + + if _is_channel_first(data_format): + mean = mean.reshape([-1, 1, 1]) + std = std.reshape([-1, 1, 1]) + return (img - mean) / std + + +def to_grayscale(img, num_output_channels=1, data_format='CHW'): + """Converts image to grayscale version of image. + + Args: + img (paddel.Tensor): Image to be converted to grayscale. + num_output_channels (int, optionl[1, 3]): + if num_output_channels = 1 : returned image is single channel + if num_output_channels = 3 : returned image is 3 channel + data_format (str, optional): Data format of img, should be 'HWC' or + 'CHW'. Default: 'CHW'. + + Returns: + paddle.Tensor: Grayscale version of the image. + """ + _assert_image_tensor(img, data_format) + + if num_output_channels not in (1, 3): + raise ValueError('num_output_channels should be either 1 or 3') + + rgb_weights = paddle.to_tensor( + [0.2989, 0.5870, 0.1140], place=img.place).astype(img.dtype) + + if _is_channel_first(data_format): + rgb_weights = rgb_weights.reshape((-1, 1, 1)) + + _c_index = _get_image_c_axis(data_format) + + img = (img * rgb_weights).sum(axis=_c_index, keepdim=True) + _shape = img.shape + _shape[_c_index] = num_output_channels + + return img.expand(_shape) + + +def _affine_grid(theta, w, h, ow, oh): + d = 0.5 + base_grid = paddle.ones((1, oh, ow, 3), dtype=theta.dtype) + + x_grid = paddle.linspace(-ow * 0.5 + d, ow * 0.5 + d - 1, ow) + base_grid[..., 0] = x_grid + y_grid = paddle.linspace(-oh * 0.5 + d, oh * 0.5 + d - 1, oh).unsqueeze_(-1) + base_grid[..., 1] = y_grid + + scaled_theta = theta.transpose( + (0, 2, 1)) / paddle.to_tensor([0.5 * w, 0.5 * h]) + output_grid = base_grid.reshape((1, oh * ow, 3)).bmm(scaled_theta) + + return output_grid.reshape((1, oh, ow, 2)) + + +def _grid_transform(img, grid, mode, fill): + if img.shape[0] > 1: + grid = grid.expand(img.shape[0], grid.shape[1], grid.shape[2], + grid.shape[3]) + + if fill is not None: + dummy = paddle.ones( + (img.shape[0], 1, img.shape[2], img.shape[3]), dtype=img.dtype) + img = paddle.concat((img, dummy), axis=1) + + img = F.grid_sample( + img, grid, mode=mode, padding_mode="zeros", align_corners=False) + + # Fill with required color + if fill is not None: + mask = img[:, -1:, :, :] # n 1 h w + img = img[:, :-1, :, :] # n c h w + mask = mask.expand_as(img) + len_fill = len(fill) if isinstance(fill, (tuple, list)) else 1 + fill_img = paddle.to_tensor(fill).reshape( + (1, len_fill, 1, 1)).expand_as(img) + + if mode == 'nearest': + mask = paddle.cast(mask < 0.5, img.dtype) + img = img * (1. - mask) + mask * fill_img + else: # 'bilinear' + img = img * mask + (1.0 - mask) * fill_img + + return img + + +def rotate(img, + angle, + interpolation='nearest', + expand=False, + center=None, + fill=None, + data_format='CHW'): + """Rotates the image by angle. + + Args: + img (paddle.Tensor): Image to be rotated. + angle (float or int): In degrees degrees counter clockwise order. + interpolation (str, optional): Interpolation method. If omitted, or if the + image has only one channel, it is set NEAREST . when use pil backend, + support method are as following: + - "nearest" + - "bilinear" + - "bicubic" + expand (bool, optional): Optional expansion flag. + If true, expands the output image to make it large enough to hold the entire rotated image. + If false or omitted, make the output image the same size as the input image. + Note that the expand flag assumes rotation around the center and no translation. + center (2-tuple, optional): Optional center of rotation. + Origin is the upper left corner. + Default is the center of the image. + fill (3-tuple or int): RGB pixel fill value for area outside the rotated image. + If int, it is used for all channels respectively. + + Returns: + paddle.Tensor: Rotated image. + + """ + + angle = -angle % 360 + img = img.unsqueeze(0) + + # n, c, h, w = img.shape + w, h = _get_image_size(img, data_format=data_format) + + img = img if data_format.lower() == 'chw' else img.transpose((0, 3, 1, 2)) + + post_trans = [0, 0] + + if center is None: + rotn_center = [0, 0] + else: + rotn_center = [(p - s * 0.5) for p, s in zip(center, [w, h])] + + angle = math.radians(angle) + matrix = [ + math.cos(angle), + math.sin(angle), + 0.0, + -math.sin(angle), + math.cos(angle), + 0.0, + ] + + matrix[2] += matrix[0] * (-rotn_center[0] - post_trans[0]) + matrix[1] * ( + -rotn_center[1] - post_trans[1]) + matrix[5] += matrix[3] * (-rotn_center[0] - post_trans[0]) + matrix[4] * ( + -rotn_center[1] - post_trans[1]) + + matrix[2] += rotn_center[0] + matrix[5] += rotn_center[1] + + matrix = paddle.to_tensor(matrix, place=img.place) + matrix = matrix.reshape((1, 2, 3)) + + if expand: + # calculate output size + corners = paddle.to_tensor( + [[-0.5 * w, -0.5 * h, 1.0], [-0.5 * w, 0.5 * h, 1.0], + [0.5 * w, 0.5 * h, 1.0], [0.5 * w, -0.5 * h, 1.0]], + place=matrix.place).astype(matrix.dtype) + + _pos = corners.reshape( + (1, -1, 3)).bmm(matrix.transpose((0, 2, 1))).reshape((1, -1, 2)) + _min = _pos.min(axis=-2).floor() + _max = _pos.max(axis=-2).ceil() + + npos = _max - _min + nw = npos[0][0] + nh = npos[0][1] + + ow, oh = int(nw.numpy()[0]), int(nh.numpy()[0]) + + else: + ow, oh = w, h + + grid = _affine_grid(matrix, w, h, ow, oh) + + out = _grid_transform(img, grid, mode=interpolation, fill=fill) + + out = out if data_format.lower() == 'chw' else out.transpose((0, 2, 3, 1)) + + return out.squeeze(0) + + +def vflip(img, data_format='CHW'): + """Vertically flips the given paddle tensor. + + Args: + img (paddle.Tensor): Image to be flipped. + data_format (str, optional): Data format of img, should be 'HWC' or + 'CHW'. Default: 'CHW'. + + Returns: + paddle.Tensor: Vertically flipped image. + + """ + _assert_image_tensor(img, data_format) + + h_axis = _get_image_h_axis(data_format) + + return img.flip(axis=[h_axis]) + + +def hflip(img, data_format='CHW'): + """Horizontally flips the given paddle.Tensor Image. + + Args: + img (paddle.Tensor): Image to be flipped. + data_format (str, optional): Data format of img, should be 'HWC' or + 'CHW'. Default: 'CHW'. + + Returns: + paddle.Tensor: Horizontall flipped image. + + """ + _assert_image_tensor(img, data_format) + + w_axis = _get_image_w_axis(data_format) + + return img.flip(axis=[w_axis]) + + +def crop(img, top, left, height, width, data_format='CHW'): + """Crops the given paddle.Tensor Image. + + Args: + img (paddle.Tensor): Image to be cropped. (0,0) denotes the top left + corner of the image. + top (int): Vertical component of the top left corner of the crop box. + left (int): Horizontal component of the top left corner of the crop box. + height (int): Height of the crop box. + width (int): Width of the crop box. + data_format (str, optional): Data format of img, should be 'HWC' or + 'CHW'. Default: 'CHW'. + Returns: + paddle.Tensor: Cropped image. + + """ + _assert_image_tensor(img, data_format) + + if _is_channel_first(data_format): + return img[:, top:top + height, left:left + width] + else: + return img[top:top + height, left:left + width, :] + + +def center_crop(img, output_size, data_format='CHW'): + """Crops the given paddle.Tensor Image and resize it to desired size. + + Args: + img (paddle.Tensor): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions + data_format (str, optional): Data format of img, should be 'HWC' or + 'CHW'. Default: 'CHW'. + Returns: + paddle.Tensor: Cropped image. + + """ + _assert_image_tensor(img, data_format) + + if isinstance(output_size, numbers.Number): + output_size = (int(output_size), int(output_size)) + + image_width, image_height = _get_image_size(img, data_format) + crop_height, crop_width = output_size + crop_top = int(round((image_height - crop_height) / 2.)) + crop_left = int(round((image_width - crop_width) / 2.)) + return crop( + img, + crop_top, + crop_left, + crop_height, + crop_width, + data_format=data_format) + + +def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'): + """ + Pads the given paddle.Tensor on all sides with specified padding mode and fill value. + + Args: + img (paddle.Tensor): Image to be padded. + padding (int|list|tuple): Padding on each border. If a single int is provided this + is used to pad all borders. If tuple of length 2 is provided this is the padding + on left/right and top/bottom respectively. If a tuple of length 4 is provided + this is the padding for the left, top, right and bottom borders + respectively. + fill (float, optional): Pixel fill value for constant fill. If a tuple of + length 3, it is used to fill R, G, B channels respectively. + This value is only used when the padding_mode is constant. Default: 0. + padding_mode: Type of padding. Should be: constant, edge, reflect or symmetric. Default: 'constant'. + + - constant: pads with a constant value, this value is specified with fill + + - edge: pads with the last value on the edge of the image + + - reflect: pads with reflection of image (without repeating the last value on the edge) + + padding [1, 2, 3, 4] with 2 elements on both sides in reflect mode + will result in [3, 2, 1, 2, 3, 4, 3, 2] + + - symmetric: pads with reflection of image (repeating the last value on the edge) + + padding [1, 2, 3, 4] with 2 elements on both sides in symmetric mode + will result in [2, 1, 1, 2, 3, 4, 4, 3] + + Returns: + paddle.Tensor: Padded image. + + """ + _assert_image_tensor(img, data_format) + + if not isinstance(padding, (numbers.Number, list, tuple)): + raise TypeError('Got inappropriate padding arg') + if not isinstance(fill, (numbers.Number, str, list, tuple)): + raise TypeError('Got inappropriate fill arg') + if not isinstance(padding_mode, str): + raise TypeError('Got inappropriate padding_mode arg') + + if isinstance(padding, (list, tuple)) and len(padding) not in [2, 4]: + raise ValueError( + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding))) + + assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \ + 'Padding mode should be either constant, edge, reflect or symmetric' + + if isinstance(padding, int): + pad_left = pad_right = pad_top = pad_bottom = padding + elif len(padding) == 2: + pad_left = pad_right = padding[0] + pad_top = pad_bottom = padding[1] + else: + pad_left = padding[0] + pad_top = padding[1] + pad_right = padding[2] + pad_bottom = padding[3] + + padding = [pad_left, pad_right, pad_top, pad_bottom] + + if padding_mode == 'edge': + padding_mode = 'replicate' + elif padding_mode == 'symmetric': + raise ValueError('Do not support symmetric mdoe') + + img = img.unsqueeze(0) + # 'constant', 'reflect', 'replicate', 'circular' + img = F.pad(img, + pad=padding, + mode=padding_mode, + value=float(fill), + data_format='N' + data_format) + + return img.squeeze(0) + + +def resize(img, size, interpolation='bilinear', data_format='CHW'): + """ + Resizes the image to given size + + Args: + input (paddle.Tensor): Image to be resized. + size (int|list|tuple): Target size of input data, with (height, width) shape. + interpolation (int|str, optional): Interpolation method. when use paddle backend, + support method are as following: + - "nearest" + - "bilinear" + - "bicubic" + - "trilinear" + - "area" + - "linear" + data_format (str, optional): paddle.Tensor format + - 'CHW' + - 'HWC' + Returns: + paddle.Tensor: Resized image. + + """ + _assert_image_tensor(img, data_format) + + if not (isinstance(size, int) or + (isinstance(size, (tuple, list)) and len(size) == 2)): + raise TypeError('Got inappropriate size arg: {}'.format(size)) + + if isinstance(size, int): + w, h = _get_image_size(img, data_format) + if (w <= h and w == size) or (h <= w and h == size): + return img + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + else: + oh, ow = size + + img = img.unsqueeze(0) + img = F.interpolate( + img, + size=(oh, ow), + mode=interpolation.lower(), + data_format='N' + data_format.upper()) + + return img.squeeze(0) diff --git a/python/paddle/vision/transforms/transforms.py b/python/paddle/vision/transforms/transforms.py index 6eeb726fcee70..00e12689c4d9f 100644 --- a/python/paddle/vision/transforms/transforms.py +++ b/python/paddle/vision/transforms/transforms.py @@ -49,6 +49,8 @@ def _get_image_size(img): return img.size elif F._is_numpy_image(img): return img.shape[:2][::-1] + elif F._is_tensor_image(img): + return img.shape[1:][::-1] # chw else: raise TypeError("Unexpected type {}".format(type(img))) @@ -690,6 +692,9 @@ def __init__(self, order=(2, 0, 1), keys=None): self.order = order def _apply_image(self, img): + if F._is_tensor_image(img): + return img.transpose(self.order) + if F._is_pil_image(img): img = np.asarray(img) From 0bb079cd47c64b411a44701af166713f1988d907 Mon Sep 17 00:00:00 2001 From: Feiyu Chan Date: Thu, 6 May 2021 17:28:54 +0800 Subject: [PATCH 036/156] avoid polluting logging's root logger (#32673) (#32706) avoid polluting logging's root logger --- .../meta_optimizers/sharding_optimizer.py | 89 ++++++++++--------- .../distributed/fleet/utils/recompute.py | 11 ++- .../fluid/incubate/fleet/utils/utils.py | 7 +- .../utils/cpp_extension/extension_utils.py | 9 +- 4 files changed, 64 insertions(+), 52 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py index 852421523b15b..db6925ace5a64 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py @@ -29,9 +29,12 @@ from paddle.fluid import layers import logging -logging.basicConfig( - format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S') +logger = logging.getLogger(__name__) +formatter = logging.Formatter( + fmt='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +ch = logging.StreamHandler() +ch.setFormatter(formatter) +logger.addHandler(ch) from functools import reduce __all__ = ["ShardingOptimizer"] @@ -136,7 +139,7 @@ def minimize_impl(self, # FIXME (JZ-LIANG) deprecated hybrid_dp if self.user_defined_strategy.sharding_configs["hybrid_dp"]: - logging.warning( + logger.warning( "[hybrid_dp] API setting is deprecated. Now when dp_degree >= 2, its will be in hybrid dp mode automatically" ) assert self.dp_degree >= 1 @@ -174,7 +177,7 @@ def minimize_impl(self, self._gradient_merge_acc_step = self.user_defined_strategy.pipeline_configs[ 'accumulate_steps'] if self._gradient_merge_acc_step > 1: - logging.info("Gradient merge in [{}], acc step = [{}]".format( + logger.info("Gradient merge in [{}], acc step = [{}]".format( self.gradient_merge_mode, self._gradient_merge_acc_step)) # optimize offload @@ -338,7 +341,7 @@ def minimize_impl(self, # opt offload should be enable while gradient merge is enable && acc_step is quite large (e.g. >> 100) # sync its memcpy could not be overlap with calc, otherwise it will slower down training severely. if self.optimize_offload: - logging.info("Sharding with optimize offload !") + logger.info("Sharding with optimize offload !") offload_helper = OffloadHelper() offload_helper.offload(main_block, startup_block) offload_helper.offload_fp32param(main_block, startup_block) @@ -641,15 +644,15 @@ def _split_program(self, block): for varname in sorted( var2broadcast_time, key=var2broadcast_time.get, reverse=True): - logging.info("Sharding broadcast: [{}] times [{}]".format( + logger.info("Sharding broadcast: [{}] times [{}]".format( var2broadcast_time[varname], varname)) for idx_ in range(len(self._segments)): - logging.info("segment [{}] :".format(idx_)) - logging.info("start op: [{}] [{}]".format(block.ops[ + logger.info("segment [{}] :".format(idx_)) + logger.info("start op: [{}] [{}]".format(block.ops[ self._segments[idx_]._start_idx].desc.type(), block.ops[ self._segments[idx_]._start_idx].desc.input_arg_names( ))) - logging.info("end op: [{}] [{}]".format(block.ops[ + logger.info("end op: [{}] [{}]".format(block.ops[ self._segments[idx_]._end_idx].desc.type(), block.ops[ self._segments[idx_]._end_idx].desc.input_arg_names())) return @@ -1108,7 +1111,7 @@ def _build_groups(self): self.dp_group_endpoints.append(self.global_endpoints[ dp_first_rank_idx + dp_offset * i]) assert self.current_endpoint in self.dp_group_endpoints - logging.info("Hybrid DP mode turn on !") + logger.info("Hybrid DP mode turn on !") else: self.dp_ring_id = -1 self.dp_rank = -1 @@ -1119,40 +1122,40 @@ def _build_groups(self): # NOTE (JZ-LIANG) when use global ring for calc global norm and dp_degree > 1, the allreduce result should be devided by dp_degree self.global_ring_id = 3 - logging.info("global word size: {}".format(self.global_word_size)) - logging.info("global rank: {}".format(self.global_rank)) - logging.info("global endpoints: {}".format(self.global_endpoints)) - logging.info("global ring id: {}".format(self.global_ring_id)) - logging.info("#####" * 6) - - logging.info("mp group size: {}".format(self.mp_degree)) - logging.info("mp rank: {}".format(self.mp_rank)) - logging.info("mp group id: {}".format(self.mp_group_id)) - logging.info("mp group endpoints: {}".format(self.mp_group_endpoints)) - logging.info("mp ring id: {}".format(self.mp_ring_id)) - logging.info("#####" * 6) - - logging.info("sharding group size: {}".format(self.sharding_degree)) - logging.info("sharding rank: {}".format(self.sharding_rank)) - logging.info("sharding group id: {}".format(self.sharding_group_id)) - logging.info("sharding group endpoints: {}".format( + logger.info("global word size: {}".format(self.global_word_size)) + logger.info("global rank: {}".format(self.global_rank)) + logger.info("global endpoints: {}".format(self.global_endpoints)) + logger.info("global ring id: {}".format(self.global_ring_id)) + logger.info("#####" * 6) + + logger.info("mp group size: {}".format(self.mp_degree)) + logger.info("mp rank: {}".format(self.mp_rank)) + logger.info("mp group id: {}".format(self.mp_group_id)) + logger.info("mp group endpoints: {}".format(self.mp_group_endpoints)) + logger.info("mp ring id: {}".format(self.mp_ring_id)) + logger.info("#####" * 6) + + logger.info("sharding group size: {}".format(self.sharding_degree)) + logger.info("sharding rank: {}".format(self.sharding_rank)) + logger.info("sharding group id: {}".format(self.sharding_group_id)) + logger.info("sharding group endpoints: {}".format( self.sharding_group_endpoints)) - logging.info("sharding ring id: {}".format(self.sharding_ring_id)) - logging.info("#####" * 6) - - logging.info("pp group size: {}".format(self.pp_degree)) - logging.info("pp rank: {}".format(self.pp_rank)) - logging.info("pp group id: {}".format(self.pp_group_id)) - logging.info("pp group endpoints: {}".format(self.pp_group_endpoints)) - logging.info("pp ring id: {}".format(self.pp_ring_id)) - logging.info("#####" * 6) - - logging.info("pure dp group size: {}".format(self.dp_degree)) - logging.info("pure dp rank: {}".format(self.dp_rank)) - logging.info("pure dp group endpoints: {}".format( + logger.info("sharding ring id: {}".format(self.sharding_ring_id)) + logger.info("#####" * 6) + + logger.info("pp group size: {}".format(self.pp_degree)) + logger.info("pp rank: {}".format(self.pp_rank)) + logger.info("pp group id: {}".format(self.pp_group_id)) + logger.info("pp group endpoints: {}".format(self.pp_group_endpoints)) + logger.info("pp ring id: {}".format(self.pp_ring_id)) + logger.info("#####" * 6) + + logger.info("pure dp group size: {}".format(self.dp_degree)) + logger.info("pure dp rank: {}".format(self.dp_rank)) + logger.info("pure dp group endpoints: {}".format( self.dp_group_endpoints)) - logging.info("pure dp ring id: {}".format(self.dp_ring_id)) - logging.info("#####" * 6) + logger.info("pure dp ring id: {}".format(self.dp_ring_id)) + logger.info("#####" * 6) return diff --git a/python/paddle/distributed/fleet/utils/recompute.py b/python/paddle/distributed/fleet/utils/recompute.py index 0dc305ec77d51..d61c3cfd1e578 100644 --- a/python/paddle/distributed/fleet/utils/recompute.py +++ b/python/paddle/distributed/fleet/utils/recompute.py @@ -19,9 +19,12 @@ import contextlib import logging -logging.basicConfig( - format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S') +logger = logging.getLogger(__name__) +formatter = logging.Formatter( + fmt='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +ch = logging.StreamHandler() +ch.setFormatter(formatter) +logger.addHandler(ch) def detach_variable(inputs): @@ -40,7 +43,7 @@ def detach_variable(inputs): def check_recompute_necessary(inputs): if not any(input_.stop_gradient == False for input_ in inputs if isinstance(input_, paddle.Tensor)): - logging.warn( + logger.warn( "[Recompute]: None of the inputs to current recompute block need grad, " "therefore there is NO need to recompute this block in backward !") diff --git a/python/paddle/fluid/incubate/fleet/utils/utils.py b/python/paddle/fluid/incubate/fleet/utils/utils.py index 79f3fb9193440..5cb4948a859d6 100644 --- a/python/paddle/fluid/incubate/fleet/utils/utils.py +++ b/python/paddle/fluid/incubate/fleet/utils/utils.py @@ -34,9 +34,12 @@ "graphviz" ] -logging.basicConfig( - format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO) logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s') +ch = logging.StreamHandler() +ch.setFormatter(formatter) +logger.addHandler(ch) persistable_vars_out_fn = "vars_persistable.log" all_vars_out_fn = "vars_all.log" diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index aa5a7ab533a28..c055084886c25 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -32,9 +32,12 @@ from ...fluid.framework import OpProtoHolder from ...sysconfig import get_include, get_lib -logging.basicConfig( - format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO) logger = logging.getLogger("utils.cpp_extension") +logger.setLevel(logging.INFO) +formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s') +ch = logging.StreamHandler() +ch.setFormatter(formatter) +logger.addHandler(ch) OS_NAME = sys.platform IS_WINDOWS = OS_NAME.startswith('win') @@ -1125,4 +1128,4 @@ def log_v(info, verbose=True): Print log information on stdout. """ if verbose: - logging.info(info) + logger.info(info) From 9a589de8da724b43e0082ea9254cba89510d13c8 Mon Sep 17 00:00:00 2001 From: chajchaj <57249073+chajchaj@users.noreply.github.com> Date: Thu, 6 May 2021 19:31:24 +0800 Subject: [PATCH 037/156] cherry-pick:change softmax_with_cross_entropy_op's parameter name from softmax_switch to use_softmax (#32750) * change parameter name from softmax_switch to use_softmax, test=develop * cherry-pick:change parameter name from softmax_switch to use_softmax, test=develop --- .../softmax_with_cross_entropy_op.cc | 7 +- .../softmax_with_cross_entropy_op.cu | 8 +- .../operators/softmax_with_cross_entropy_op.h | 16 ++-- .../test_softmax_with_cross_entropy_op.py | 78 +++++++++---------- python/paddle/nn/functional/loss.py | 6 +- 5 files changed, 56 insertions(+), 59 deletions(-) diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.cc b/paddle/fluid/operators/softmax_with_cross_entropy_op.cc index e58b39252ce5f..fbaf76d4e7cd8 100644 --- a/paddle/fluid/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.cc @@ -55,7 +55,7 @@ class SoftmaxWithCrossEntropyOpMaker "the given labels as soft labels.") .SetDefault(false); AddAttr( - "softmax_switch", + "use_softmax", "(bool, default: true), A flag to indicate whether to do softmax ") .SetDefault(true); AddAttr( @@ -320,7 +320,6 @@ REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy_grad, REGISTER_OP_VERSION(softmax_with_cross_entropy) .AddCheckpoint( R"ROC( - Add a new attribute [softmax_switch] )ROC", + Add a new attribute [use_softmax] )ROC", paddle::framework::compatible::OpVersionDesc().NewAttr( - "softmax_switch", "A flag to indicate whether to do softmax", - true)); + "use_softmax", "A flag to indicate whether to do softmax", true)); diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu index 140059256c3cc..4aec4c1742279 100644 --- a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu +++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu @@ -772,10 +772,10 @@ class SoftmaxWithCrossEntropyCUDAKernel : public framework::OpKernel { platform::is_gpu_place(context.GetPlace()), true, platform::errors::Unavailable("softmax_with_cross_entropy operator's " "CUDA kernel only runs on GPU device.")); - const bool softmax_switch = context.Attr("softmax_switch"); + const bool use_softmax = context.Attr("use_softmax"); // do not with softmax op, and input is softmax - if (!softmax_switch) { + if (!use_softmax) { const Tensor* softmax = context.Input("Logits"); const Tensor* labels = context.Input("Label"); Tensor* softmax_out = context.Output("Softmax"); @@ -925,10 +925,10 @@ class SoftmaxWithCrossEntropyGradCUDAKernel : public framework::OpKernel { int block = 512; auto stream = context.cuda_device_context().stream(); auto ignore_index = context.Attr("ignore_index"); - auto softmax_switch = context.Attr("softmax_switch"); + auto use_softmax = context.Attr("use_softmax"); // do not with softmax op, and input is softmax - if (!softmax_switch) { + if (!use_softmax) { if (context.Attr("soft_label")) { int grid = (n * d + block - 1) / block; const T* label_data = labels->data(); diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.h b/paddle/fluid/operators/softmax_with_cross_entropy_op.h index 55b811cbe31e4..74316841a13b1 100644 --- a/paddle/fluid/operators/softmax_with_cross_entropy_op.h +++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.h @@ -31,10 +31,10 @@ class SoftmaxWithCrossEntropyKernel : public framework::OpKernel { PADDLE_ENFORCE_EQ( platform::is_cpu_place(context.GetPlace()), true, platform::errors::Unimplemented("This kernel only runs on CPU.")); - const bool softmax_switch = context.Attr("softmax_switch"); + const bool use_softmax = context.Attr("use_softmax"); // do not with softmax op, and input is softmax - if (!softmax_switch) { + if (!use_softmax) { const Tensor* softmax = context.Input("Logits"); const Tensor* labels = context.Input("Label"); Tensor* softmax_out = context.Output("Softmax"); @@ -113,9 +113,9 @@ class SoftmaxWithCrossEntropyGradKernel : public framework::OpKernel { context.Output(framework::GradVarName("Logits")); const Tensor* softmax = context.Input("Softmax"); - const bool softmax_switch = context.Attr("softmax_switch"); + const bool use_softmax = context.Attr("use_softmax"); - if (logit_grad != softmax || !softmax_switch) { + if (logit_grad != softmax || !use_softmax) { framework::TensorCopy(*softmax, context.GetPlace(), context.device_context(), logit_grad); } @@ -138,8 +138,8 @@ class SoftmaxWithCrossEntropyGradKernel : public framework::OpKernel { auto logit_grad_mat = framework::EigenMatrix::From(logit_grad_2d); auto& place = *context.template device_context() .eigen_device(); - if (!softmax_switch) { - // softmax_switch step1 + if (!use_softmax) { + // use_softmax step1 if (soft_label) { auto lbl_mat = framework::EigenMatrix::From(labels_2d); logit_grad_mat.device(place) = @@ -148,7 +148,7 @@ class SoftmaxWithCrossEntropyGradKernel : public framework::OpKernel { out_grad_mat.broadcast(Eigen::DSizes(1, axis_dim)) * logit_grad_mat; } - // softmax_switch step2 + // use_softmax step2 else { const int64_t* label_data = labels->data(); T* logit_grad_data = logit_grad->data(); @@ -181,7 +181,7 @@ class SoftmaxWithCrossEntropyGradKernel : public framework::OpKernel { return; } - // for softmax_switch=False, continue + // for use_softmax=False, continue if (soft_label) { // when soft_label = True, ignore_index is not supported diff --git a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py index e1f5ecf268304..e754999d5d205 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py @@ -56,7 +56,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.shape = [41, 37] - self.softmax_switch = True + self.use_softmax = True def setUp(self): self.initParams() @@ -77,7 +77,7 @@ def setUp(self): loss = cross_entropy(softmax, labels, self.soft_label, self.axis, self.ignore_index) - if self.softmax_switch == False: + if self.use_softmax == False: self.inputs = {"Logits": softmax, "Label": labels} else: self.inputs = {"Logits": logits, "Label": labels} @@ -90,7 +90,7 @@ def setUp(self): "numeric_stable_mode": self.numeric_stable_mode, "soft_label": self.soft_label, "ignore_index": self.ignore_index, - "softmax_switch": self.softmax_switch, + "use_softmax": self.use_softmax, } if self.axis != -1: @@ -117,7 +117,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_1D( @@ -130,7 +130,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" ############################################################################## @@ -146,7 +146,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis2( @@ -159,7 +159,7 @@ def initParams(self): self.axis = 1 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis3( @@ -172,7 +172,7 @@ def initParams(self): self.axis = 2 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis4( @@ -185,7 +185,7 @@ def initParams(self): self.axis = 3 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" ############################################################################## @@ -207,7 +207,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis2( @@ -220,7 +220,7 @@ def initParams(self): self.axis = 1 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis3( @@ -233,7 +233,7 @@ def initParams(self): self.axis = 2 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis4( @@ -246,7 +246,7 @@ def initParams(self): self.axis = 3 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" ############################################################################## @@ -268,7 +268,7 @@ def initParams(self): self.axis = -1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_Ignore_Axis( @@ -281,7 +281,7 @@ def initParams(self): self.axis = 1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore( @@ -294,7 +294,7 @@ def initParams(self): self.axis = -1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore_Axis3( @@ -307,7 +307,7 @@ def initParams(self): self.axis = 2 self.ignore_index = 2 self.shape = [3, 5, 7, 11] - self.softmax_switch = False #default is true, means "with softmax" + self.use_softmax = False #default is true, means "with softmax" ############################################################################## @@ -324,7 +324,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True @unittest.skipIf(not core.is_compiled_with_cuda(), @@ -403,7 +403,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.shape = [41, 37] - self.softmax_switch = True + self.use_softmax = True def test_check_output(self): self.check_output() @@ -429,7 +429,7 @@ def initParams(self): self.ignore_index = 5 self.axis = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOp3NoCudnn(TestSoftmaxWithCrossEntropyOp3): @@ -441,7 +441,7 @@ def initParams(self): self.ignore_index = 4 self.axis = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpAxis1(TestSoftmaxWithCrossEntropyOp): @@ -458,7 +458,7 @@ def initParams(self): self.axis = 0 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpAxis2(TestSoftmaxWithCrossEntropyOp): @@ -475,7 +475,7 @@ def initParams(self): self.axis = 1 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpAxis3(TestSoftmaxWithCrossEntropyOp): @@ -492,7 +492,7 @@ def initParams(self): self.axis = 2 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpAxis4(TestSoftmaxWithCrossEntropyOp): @@ -509,7 +509,7 @@ def initParams(self): self.axis = 3 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpAxisDimEqualOne( @@ -527,7 +527,7 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.shape = [3, 5, 7, 1] - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis1( @@ -540,7 +540,7 @@ def initParams(self): self.axis = 0 self.ignore_index = -1 self.dtype = np.float16 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis2( @@ -553,7 +553,7 @@ def initParams(self): self.axis = 1 self.ignore_index = -1 self.dtype = np.float16 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis3( @@ -566,7 +566,7 @@ def initParams(self): self.axis = 2 self.ignore_index = -1 self.dtype = np.float16 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpSoftLabelAxis1( @@ -579,7 +579,7 @@ def initParams(self): self.axis = 0 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpSoftLabelAxis2( @@ -592,7 +592,7 @@ def initParams(self): self.axis = 1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpSoftLabelAxis3( @@ -605,7 +605,7 @@ def initParams(self): self.axis = 2 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpSoftLabelAxis4( @@ -618,7 +618,7 @@ def initParams(self): self.axis = 3 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis1( @@ -631,7 +631,7 @@ def initParams(self): self.ignore_index = 1 self.axis = 0 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis2( @@ -644,7 +644,7 @@ def initParams(self): self.ignore_index = 0 self.axis = 1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis3( @@ -657,7 +657,7 @@ def initParams(self): self.ignore_index = 3 self.axis = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis4( @@ -670,7 +670,7 @@ def initParams(self): self.ignore_index = 3 self.axis = 3 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpBoundary0(TestSoftmaxWithCrossEntropyOp): @@ -688,7 +688,7 @@ def initParams(self): self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.logits = np.full(self.shape, -500.0).astype(self.dtype) - self.softmax_switch = True + self.use_softmax = True class TestSoftmaxWithCrossEntropyOpBoundary1(TestSoftmaxWithCrossEntropyOp): @@ -707,7 +707,7 @@ def initParams(self): self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.logits = np.full(self.shape, 1000.0).astype(self.dtype) self.logits[:, :, 0, :] = -1000.0 - self.softmax_switch = True + self.use_softmax = True if __name__ == "__main__": diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index 31ffb91f30dca..b89da3d82e379 100755 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -1371,8 +1371,6 @@ def cross_entropy(input, "should be '-100', but received %s, which is not allowed." % ignore_index) - softmax_switch = use_softmax - input_dims = len(list(input.shape)) label_dims = len(list(label.shape)) if input_dims - 1 != label_dims and input_dims != label_dims: @@ -1385,7 +1383,7 @@ def cross_entropy(input, _, out = core.ops.softmax_with_cross_entropy( input, label, 'soft_label', soft_label, 'ignore_index', ignore_index, 'numeric_stable_mode', True, 'axis', axis, - 'softmax_switch', softmax_switch) + 'use_softmax', use_softmax) if weight is not None: @@ -1467,7 +1465,7 @@ def cross_entropy(input, 'ignore_index': ignore_index, 'numeric_stable_mode': True, 'axis': axis, - 'softmax_switch': softmax_switch + 'use_softmax': use_softmax } helper = LayerHelper('softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=input.dtype) From 21448525229896dfbbd9fe2b669280135fb446c0 Mon Sep 17 00:00:00 2001 From: jakpiase <62569058+jakpiase@users.noreply.github.com> Date: Thu, 6 May 2021 14:06:38 +0200 Subject: [PATCH 038/156] [CHERRY-PICK] Reduce grad fix cherrypick (#32742) * base changes for fix * minor change * fix for bwd kernel * removed unnecessary import * implemented reviewers suggestions * CI fix --- .../mkldnn/reduce_mean_mkldnn_op.cc | 3 +- .../reduce_ops/mkldnn/reduce_mkldnn_op.h | 90 ++++++++++++------- .../reduce_ops/mkldnn/reduce_sum_mkldnn_op.cc | 3 +- paddle/fluid/operators/reduce_ops/reduce_op.h | 25 ++---- paddle/fluid/platform/mkldnn_reuse.h | 31 +++---- 5 files changed, 79 insertions(+), 73 deletions(-) diff --git a/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mean_mkldnn_op.cc b/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mean_mkldnn_op.cc index 33daeea8599c6..dfba933940bd0 100644 --- a/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mean_mkldnn_op.cc +++ b/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mean_mkldnn_op.cc @@ -45,7 +45,8 @@ class ReduceMeanGradMKLDNNKernel : public ReduceGradMKLDNNKernel { number_of_elements = input_x->numel(); } - this->RunKernel(ctx, dnnl::algorithm::binary_add, 0.0f, + this->RunKernel(ctx, dnnl::algorithm::binary_add, + dnnl::algorithm::reduction_mean, 0.0f, 1.0L / number_of_elements); } }; diff --git a/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mkldnn_op.h b/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mkldnn_op.h index 58416f479c043..40cd3ba974f04 100644 --- a/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mkldnn_op.h +++ b/paddle/fluid/operators/reduce_ops/mkldnn/reduce_mkldnn_op.h @@ -21,6 +21,27 @@ using paddle::framework::LoDTensor; using paddle::framework::Tensor; using platform::to_void_cast; +inline std::vector CalculateReducedDims(const Tensor* input, + const Tensor* output, + std::vector& reduce_dims, + bool reduce_all, + bool keep_dim) { + if (keep_dim) return framework::vectorize(output->dims()); + + if (reduce_all) + return std::vector(framework::vectorize(input->dims()).size(), 1); + + std::vector output_dims(framework::vectorize(input->dims())); + for (size_t i = 0; i < reduce_dims.size(); ++i) { + reduce_dims[i] = (reduce_dims[i] >= 0) + ? reduce_dims[i] + : input->dims().size() + reduce_dims[i]; + output_dims[reduce_dims[i]] = 1; + } + + return output_dims; +} + template class ReduceMKLDNNKernel : public framework::OpKernel { public: @@ -37,9 +58,8 @@ class ReduceMKLDNNKernel : public framework::OpKernel { bool reduce_all = ctx.Attr("reduce_all"); bool keep_dim = ctx.Attr("keep_dim"); - std::vector output_dims = - CalculateOutputDims(input, output, reduce_dims, reduce_all, keep_dim); - + auto output_dims = + CalculateReducedDims(input, output, reduce_dims, reduce_all, keep_dim); auto input_dims = framework::vectorize(input->dims()); auto& astream = platform::MKLDNNDeviceContext::tls().get_stream(); @@ -96,53 +116,63 @@ class ReduceMKLDNNKernel : public framework::OpKernel { paddle::framework::vectorize(output->dims())))); } } - - private: - std::vector CalculateOutputDims(const Tensor* input, - const Tensor* output, - std::vector& reduce_dims, - bool reduce_all, - bool keep_dim) const { - if (keep_dim) return framework::vectorize(output->dims()); - - if (reduce_all) - return std::vector(framework::vectorize(input->dims()).size(), - 1); - - std::vector output_dims(framework::vectorize(input->dims())); - for (size_t i = 0; i < reduce_dims.size(); ++i) { - reduce_dims[i] = (reduce_dims[i] >= 0) - ? reduce_dims[i] - : input->dims().size() + reduce_dims[i]; - output_dims[reduce_dims[i]] = 1; - } - - return output_dims; - } }; template class ReduceGradMKLDNNKernel : public framework::OpKernel { public: void RunKernel(const framework::ExecutionContext& ctx, - dnnl::algorithm binary_type, float scale_x, - float scale_y) const { + dnnl::algorithm binary_type, dnnl::algorithm reduction_type, + float scale_x, float scale_y) const { const auto& dev_ctx = ctx.template device_context(); const auto& onednn_engine = dev_ctx.GetEngine(); + bool keep_dim = ctx.Attr("keep_dim"); + bool reduce_all = ctx.Attr("reduce_all"); auto dims = ctx.Attr>("dim"); auto* input_dy = ctx.Input(framework::GradVarName("Out")); auto* output_dx = ctx.Output(framework::GradVarName("X")); + mkldnn::memory::format_tag x_format_tag; + auto input_dims = + CalculateReducedDims(output_dx, input_dy, dims, reduce_all, keep_dim); + + if (input_dims != framework::vectorize(output_dx->dims())) { + const std::string key_pd = + platform::CreateKey( + dev_ctx, framework::vectorize(output_dx->dims()), + ctx.InputName("X"), + (std::to_string(static_cast(reduction_type)))) + + "@fwd_pd"; + std::shared_ptr fwd_pd = + std::static_pointer_cast( + dev_ctx.GetBlob(key_pd)); + + PADDLE_ENFORCE_NOT_NULL( + fwd_pd, platform::errors::Unavailable( + "Forward primitive descriptor is not available in %s op, " + "cannot deduce memory format tag", + ctx.Type())); + + x_format_tag = platform::GetMKLDNNFormat(fwd_pd->src_desc()); + + PADDLE_ENFORCE_NE(x_format_tag, mkldnn::memory::format_tag::undef, + platform::errors::InvalidArgument( + "Cannot deduce format tag for %s op", ctx.Type())); + } else { // fwd descriptor not available because reorder was used instead + // of reduction + x_format_tag = getPlainFormatTag(output_dx); + } + output_dx->mutable_data(ctx.GetPlace()); - output_dx->set_format(getPlainFormatTag(output_dx)); + output_dx->set_format(x_format_tag); output_dx->set_layout(input_dy->layout()); platform::BroadcastDataMKLDNNHandler handler( binary_type, dev_ctx, onednn_engine, ctx.GetPlace(), output_dx, input_dy, scale_x, scale_y, - ctx.InputName(framework::GradVarName("Out"))); + ctx.InputName(framework::GradVarName("Out")), input_dims); const auto src_dx_memory = handler.AcquireSrcMemory(output_dx); const auto src_dy_memory = handler.AcquireSecondSrcMemory(input_dy); diff --git a/paddle/fluid/operators/reduce_ops/mkldnn/reduce_sum_mkldnn_op.cc b/paddle/fluid/operators/reduce_ops/mkldnn/reduce_sum_mkldnn_op.cc index e62edcf559677..3f92d39ede1ae 100644 --- a/paddle/fluid/operators/reduce_ops/mkldnn/reduce_sum_mkldnn_op.cc +++ b/paddle/fluid/operators/reduce_ops/mkldnn/reduce_sum_mkldnn_op.cc @@ -29,7 +29,8 @@ template class ReduceSumGradMKLDNNKernel : public ReduceGradMKLDNNKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - this->RunKernel(ctx, dnnl::algorithm::binary_add, 0.0f, 1.0f); + this->RunKernel(ctx, dnnl::algorithm::binary_add, + dnnl::algorithm::reduction_sum, 0.0f, 1.0f); } }; diff --git a/paddle/fluid/operators/reduce_ops/reduce_op.h b/paddle/fluid/operators/reduce_ops/reduce_op.h index 913d941df8810..390c4d9709a60 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_op.h +++ b/paddle/fluid/operators/reduce_ops/reduce_op.h @@ -559,8 +559,11 @@ class ReduceGradOp : public framework::OperatorWithKernel { protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { - auto input_data_type = OperatorWithKernel::IndicateVarDataType( - ctx, framework::GradVarName("Out")); + int in_dtype = ctx.Attr("in_dtype"); + auto input_data_type = + (in_dtype >= 0) ? static_cast(in_dtype) + : OperatorWithKernel::IndicateVarDataType( + ctx, framework::GradVarName("Out")); #ifdef PADDLE_WITH_MKLDNN auto CanMKLDNNReduceGradBeUsed = [&]() { @@ -568,18 +571,6 @@ class ReduceGradOp : public framework::OperatorWithKernel { if (dx_dims.size() > 5) return false; // max 5D tensor is supported - if (ctx.Attr("reduce_all") || - ((int)ctx.Attr>("dim").size() == dx_dims.size())) - return true; - - auto dy_dims = ctx.Input(framework::GradVarName("Out"))->dims(); - - // Subtensor must be on rightmost part of the bigger tensor - for (int i = 0; i < dy_dims.size(); ++i) { - if (dx_dims[dx_dims.size() - dy_dims.size() + i] != dy_dims[i]) { - return false; - } - } return true; }; if (this->CanMKLDNNBeUsed(ctx, input_data_type) && @@ -590,12 +581,6 @@ class ReduceGradOp : public framework::OperatorWithKernel { } #endif - int in_dtype = ctx.Attr("in_dtype"); - if (in_dtype >= 0) { - return framework::OpKernelType( - static_cast(in_dtype), - ctx.GetPlace()); - } return framework::OpKernelType(input_data_type, ctx.GetPlace()); } }; diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index 54efa55cc4cd9..f1eb1f9636375 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -639,7 +639,8 @@ class BroadcastDataMKLDNNHandler const mkldnn::engine engine, platform::Place cpu_place, const Tensor* x, const Tensor* y, float scale_x, float scale_y, - const std::string& uniq_name) + const std::string& uniq_name, + std::vector& input_dims) : platform::MKLDNNHandlerT( dev_ctx, engine, cpu_place, platform::CreateKey(dev_ctx, framework::vectorize(x->dims()), @@ -659,24 +660,12 @@ class BroadcastDataMKLDNNHandler y->format(), MKLDNNMemoryFormat::undef, platform::errors::InvalidArgument("Wrong format set for Y tensor.")); - auto src1_tz = framework::vectorize(y->dims()); const auto src0_tz = framework::vectorize(x->dims()); - // GetExpectedKernelType checks if smaller vector is a subvector with all - // the dims in correct order on the rightmost part of the bigger vector, - // i.e. a correct vector for broadcasting: - // x = 5, 7, 3, 2, 4, 8 - // y = 4, 8 - src1_tz.reserve(src0_tz.size()); - - for (size_t i = src1_tz.size(); i < src0_tz.size(); ++i) { - src1_tz.insert(src1_tz.begin(), 1L); - } - const auto src0_md = dnnl::memory::desc( src0_tz, platform::MKLDNNGetDataType(), x->format()); const auto src1_md = dnnl::memory::desc( - src1_tz, platform::MKLDNNGetDataType(), x->format()); + input_dims, platform::MKLDNNGetDataType(), x->format()); dnnl::primitive_attr attributes; attributes.set_scales(DNNL_ARG_SRC_0, 0, {scale_x}); @@ -711,7 +700,7 @@ class ReductionMKLDNNHandler const mkldnn::engine engine, platform::Place cpu_place, const Tensor* x, const Tensor* y, const std::string& uniq_name, - std::vector output_dims) + std::vector y_tz) : platform::MKLDNNHandlerT( dev_ctx, engine, cpu_place, platform::CreateKey(dev_ctx, framework::vectorize(x->dims()), @@ -725,14 +714,14 @@ class ReductionMKLDNNHandler x->format(), MKLDNNMemoryFormat::undef, platform::errors::InvalidArgument("Wrong format set for X tensor.")); - const auto src_tz = framework::vectorize(x->dims()); + const auto x_tz = framework::vectorize(x->dims()); - const auto src_md = dnnl::memory::desc( - src_tz, platform::MKLDNNGetDataType(), x->format()); - const auto dst_md = memory::desc( - output_dims, platform::MKLDNNGetDataType(), x->format()); + const auto x_md = dnnl::memory::desc( + x_tz, platform::MKLDNNGetDataType(), x->format()); + const auto y_md = + memory::desc(y_tz, platform::MKLDNNGetDataType(), x->format()); - this->AcquireForwardPrimitiveDescriptor(algo, src_md, dst_md, p, eps); + this->AcquireForwardPrimitiveDescriptor(algo, x_md, y_md, p, eps); } } }; From f3436af1ba8403f59fba592857e7582713a30011 Mon Sep 17 00:00:00 2001 From: Adam Osewski Date: Thu, 6 May 2021 14:07:14 +0200 Subject: [PATCH 039/156] [cherry-pick] Sum kernel for CPU supporting BF16 and SelectedRows (#32631) (#32755) --- paddle/fluid/operators/math/blas_impl.h | 19 +++++ .../operators/math/selected_rows_functor.cc | 40 +++++------ paddle/fluid/operators/sum_op.cc | 2 + .../fluid/tests/unittests/test_sgd_op_bf16.py | 9 +-- .../fluid/tests/unittests/test_sum_op.py | 71 +++++++++++++++++++ 5 files changed, 115 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/operators/math/blas_impl.h b/paddle/fluid/operators/math/blas_impl.h index 64b533de098ca..05d42f02c1003 100644 --- a/paddle/fluid/operators/math/blas_impl.h +++ b/paddle/fluid/operators/math/blas_impl.h @@ -15,6 +15,7 @@ #ifdef PADDLE_WITH_MKLML #include #endif + #include #include #include @@ -28,6 +29,19 @@ namespace paddle { namespace operators { namespace math { +namespace detail { + +template +static void axpy(int n, const T alpha, const T *x, const int incx, T *y, + const int incy) { + // Y = Y + alpha * X + while (n-- > 0) { + *y += alpha * *x; + y = y + incy; + x = x + incx; + } +} +} // namespace detail template struct CBlas; @@ -43,6 +57,11 @@ struct CBlas { template <> struct CBlas { + template + static void AXPY(ARGS... args) { + detail::axpy(args...); + } + template static void VCOPY(ARGS... args) { PADDLE_THROW(platform::errors::Unimplemented( diff --git a/paddle/fluid/operators/math/selected_rows_functor.cc b/paddle/fluid/operators/math/selected_rows_functor.cc index f7b16453e0133..b9a1854a66118 100644 --- a/paddle/fluid/operators/math/selected_rows_functor.cc +++ b/paddle/fluid/operators/math/selected_rows_functor.cc @@ -285,6 +285,8 @@ template struct SelectedRowsAddToTensor; template struct SelectedRowsAddToTensor; template struct SelectedRowsAddToTensor; template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; // This is a separated namespace for manipulate SelectedRows typed // data. Like merge duplicated rows, adding two SelectedRows etc. @@ -294,21 +296,17 @@ template struct SelectedRowsAddToTensor; // add or mul. namespace scatter { -template -typename std::enable_if< - std::is_floating_point::value && - std::is_same::value>::type -elementwise_add_to(const DeviceContext& ctx, BlasT* blas, - size_t data_len, const T* in, T* out) { - blas->AXPY(data_len, 1., in, out); +template +typename std::enable_if::value>::type +elementwise_add_to(BlasT* blas, size_t data_len, + const T* in, T* out) { + blas->AXPY(data_len, T(1.f), in, out); } -template -typename std::enable_if< - !std::is_floating_point::value && - std::is_same::value>::type -elementwise_add_to(const DeviceContext& ctx, BlasT* blas, - size_t data_len, const T* in, T* out) { +template +typename std::enable_if::value>::type elementwise_add_to( + BlasT* blas, size_t data_len, const T* in, + T* out) { for (size_t i = 0; i < data_len; i++) { out[i] += in[i]; } @@ -412,7 +410,7 @@ struct MergeAdd { out.set_rows(merge_rows); math::SetConstant constant_functor; - constant_functor(context, out.mutable_value(), 0.0); + constant_functor(context, out.mutable_value(), static_cast(0.f)); std::unordered_map rows_to_id; for (size_t i = 0; i < merge_rows.size(); ++i) { @@ -429,9 +427,9 @@ struct MergeAdd { for (size_t i = 0; i < input_rows.size(); i++) { size_t out_i = rows_to_id[input_rows[i]]; - elementwise_add_to( - context, &blas, static_cast(input_width), - &input_data[i * input_width], &out_data[out_i * input_width]); + elementwise_add_to(&blas, static_cast(input_width), + &input_data[i * input_width], + &out_data[out_i * input_width]); } } } @@ -524,9 +522,9 @@ struct MergeAverage { for (size_t i = 0; i < input_rows.size(); i++) { size_t out_i = rows_to_id[input_rows[i]]; - elementwise_add_to( - context, &blas, static_cast(input_width), - &input_data[i * input_width], &out_data[out_i * input_width]); + elementwise_add_to(&blas, static_cast(input_width), + &input_data[i * input_width], + &out_data[out_i * input_width]); } } size_t input_width_cast = static_cast(input_width); @@ -547,6 +545,8 @@ template struct MergeAdd; template struct MergeAdd; +template struct MergeAdd; template struct MergeAverage; template struct MergeAverage; diff --git a/paddle/fluid/operators/sum_op.cc b/paddle/fluid/operators/sum_op.cc index 741f86f35848b..0f520adba57a2 100644 --- a/paddle/fluid/operators/sum_op.cc +++ b/paddle/fluid/operators/sum_op.cc @@ -326,4 +326,6 @@ REGISTER_OP_CPU_KERNEL( sum, ops::SumKernel, ops::SumKernel, ops::SumKernel, + ops::SumKernel, ops::SumKernel); diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py b/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py index 0717ec80f6a13..fa8ff4effcfd3 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py @@ -76,8 +76,7 @@ def create_sparse_grad_var(self, scope, place, height, rows, row_numel): grad_selected_rows = scope.var('Grad').get_selected_rows() grad_selected_rows.set_height(height) grad_selected_rows.set_rows(rows) - # grad_array = np.random.random((len(rows), row_numel)).astype('float32') - grad_array = np.full((len(rows), row_numel), 2, np.float32) + grad_array = np.random.random((len(rows), row_numel)).astype('float32') np_array_bf16 = convert_float_to_uint16(grad_array) grad_tensor = grad_selected_rows.get_tensor() @@ -87,8 +86,7 @@ def create_sparse_grad_var(self, scope, place, height, rows, row_numel): def create_dense_param_var(self, scope, place, height, width): param_tensor = scope.var('Param').get_tensor() - # param_array = np.random.random((height, width)).astype('float32') - param_array = np.full((height, width), 5, np.float32) + param_array = np.random.random((height, width)).astype('float32') param_array_bf16 = convert_float_to_uint16(param_array) param_tensor.set(param_array_bf16, place) @@ -109,8 +107,7 @@ def create_sparse_param_var(self, scope, place, height, rows, row_numel): def create_dense_lr_var(self, scope, place): lr_tensor = scope.var('LearningRate').get_tensor() - # lr_value = np.random.uniform() - lr_value = 2 + lr_value = np.random.uniform() lr_array = np.full((1), lr_value, np.float32) lr_array_bf16 = convert_float_to_uint16(lr_array) lr_tensor.set(lr_array_bf16, place) diff --git a/python/paddle/fluid/tests/unittests/test_sum_op.py b/python/paddle/fluid/tests/unittests/test_sum_op.py index 35dc92ffb08c6..f9e40cf8133d7 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_op.py @@ -18,9 +18,12 @@ import numpy as np from op_test import OpTest import paddle +from paddle import enable_static import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, convert_float_to_uint16, convert_uint16_to_float) class TestSumOp(OpTest): @@ -141,6 +144,73 @@ def test_w_is_selected_rows(self): self.check_with_place(place, inplace) +class TestSelectedRowsSumOpInt(TestSelectedRowsSumOp): + def init_kernel_type(self): + self.dtype = np.int32 + + +@unittest.skipIf(not core.supports_bfloat16(), + 'place does not support BF16 evaluation') +class TestSelectedRowsSumBF16Op(TestSelectedRowsSumOp): + def setUp(self): + self.height = 10 + self.row_numel = 12 + self.rows = [0, 1, 2, 3, 4, 5, 6] + self.dtype = np.uint16 + self.init_kernel_type() + np.random.seed(12345) + self.data = np.random.random((len(self.rows), + self.row_numel)).astype(np.float32) + + def _get_array(self, rows, row_numel): + if len(rows) > 0: + return convert_float_to_uint16(self.data) + else: + return np.ndarray((0, row_numel), dtype=self.dtype) + + def check_input_and_optput(self, + scope, + place, + inplace, + w1_has_data=False, + w2_has_data=False, + w3_has_data=False): + + self.create_selected_rows(scope, place, "W1", w1_has_data) + self.create_selected_rows(scope, place, "W2", w2_has_data) + self.create_selected_rows(scope, place, "W3", w3_has_data) + + # create Out Variable + if inplace: + out_var_name = "W1" + else: + out_var_name = "Out" + out = scope.var(out_var_name).get_selected_rows() + + # create and run sum operator + sum_op = Operator("sum", X=["W1", "W2", "W3"], Out=out_var_name) + sum_op.run(scope, place) + + has_data_w_num = 0 + for has_data in [w1_has_data, w2_has_data, w3_has_data]: + if has_data: + has_data_w_num += 1 + + if has_data_w_num > 0: + self.assertEqual(len(out.rows()), 7) + out_bf16 = np.array(out.get_tensor()) + out_fp32 = convert_uint16_to_float(out_bf16) + ref_fp32 = convert_uint16_to_float( + self._get_array(self.rows, self.row_numel)) * has_data_w_num + np.testing.assert_allclose(out_fp32, ref_fp32, atol=0, rtol=0.95e-2) + else: + self.assertEqual(len(out.rows()), 0) + + def test_w_is_selected_rows(self): + for inplace in [True, False]: + self.check_with_place(core.CPUPlace(), inplace) + + class TestLoDTensorAndSelectedRowsOp(TestSelectedRowsSumOp): def setUp(self): self.height = 10 @@ -324,4 +394,5 @@ def test_list_of_none_input(): create_test_sum_fp16_class(TestLoDTensorAndSelectedRowsOp) if __name__ == "__main__": + enable_static() unittest.main() From 4f06cd17d43f48bb15a28ca63cbdf35e3db49e7d Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Thu, 6 May 2021 20:39:48 +0800 Subject: [PATCH 040/156] Pick revert data generator (#32700) * revert data_generator * add setup.py --- .../fluid/incubate/data_generator/__init__.py | 343 ++++++++++++++++++ python/setup.py.in | 1 + 2 files changed, 344 insertions(+) create mode 100644 python/paddle/fluid/incubate/data_generator/__init__.py diff --git a/python/paddle/fluid/incubate/data_generator/__init__.py b/python/paddle/fluid/incubate/data_generator/__init__.py new file mode 100644 index 0000000000000..b7c1c9863b080 --- /dev/null +++ b/python/paddle/fluid/incubate/data_generator/__init__.py @@ -0,0 +1,343 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +__all__ = ['MultiSlotDataGenerator', 'MultiSlotStringDataGenerator'] + + +class DataGenerator(object): + """ + DataGenerator is a general Base class for user to inherit + A user who wants to define his/her own python processing logic + with paddle.fluid.dataset should inherit this class. + """ + + def __init__(self): + self._proto_info = None + self.batch_size_ = 32 + + def _set_line_limit(self, line_limit): + if not isinstance(line_limit, int): + raise ValueError("line_limit%s must be in int type" % + type(line_limit)) + if line_limit < 1: + raise ValueError("line_limit can not less than 1") + self._line_limit = line_limit + + def set_batch(self, batch_size): + ''' + Set batch size of current DataGenerator + This is necessary only if a user wants to define generator_batch + + Example: + .. code-block:: python + import paddle.fluid.incubate.data_generator as dg + class MyData(dg.DataGenerator): + def generate_sample(self, line): + def local_iter(): + int_words = [int(x) for x in line.split()] + yield ("words", int_words) + return local_iter + def generate_batch(self, samples): + def local_iter(): + for s in samples: + yield ("words", s[1].extend([s[1][0]])) + mydata = MyData() + mydata.set_batch(128) + + ''' + self.batch_size_ = batch_size + + def run_from_memory(self): + ''' + This function generator data from memory, it is usually used for + debug and benchmarking + Example: + .. code-block:: python + import paddle.fluid.incubate.data_generator as dg + class MyData(dg.DataGenerator): + def generate_sample(self, line): + def local_iter(): + yield ("words", [1, 2, 3, 4]) + return local_iter + mydata = MyData() + mydata.run_from_memory() + ''' + batch_samples = [] + line_iter = self.generate_sample(None) + for user_parsed_line in line_iter(): + if user_parsed_line == None: + continue + batch_samples.append(user_parsed_line) + if len(batch_samples) == self.batch_size_: + batch_iter = self.generate_batch(batch_samples) + for sample in batch_iter(): + sys.stdout.write(self._gen_str(sample)) + batch_samples = [] + if len(batch_samples) > 0: + batch_iter = self.generate_batch(batch_samples) + for sample in batch_iter(): + sys.stdout.write(self._gen_str(sample)) + + def run_from_stdin(self): + ''' + This function reads the data row from stdin, parses it with the + process function, and further parses the return value of the + process function with the _gen_str function. The parsed data will + be wrote to stdout and the corresponding protofile will be + generated. + Example: + + .. code-block:: python + import paddle.fluid.incubate.data_generator as dg + class MyData(dg.DataGenerator): + def generate_sample(self, line): + def local_iter(): + int_words = [int(x) for x in line.split()] + yield ("words", [int_words]) + return local_iter + mydata = MyData() + mydata.run_from_stdin() + ''' + batch_samples = [] + for line in sys.stdin: + line_iter = self.generate_sample(line) + for user_parsed_line in line_iter(): + if user_parsed_line == None: + continue + batch_samples.append(user_parsed_line) + if len(batch_samples) == self.batch_size_: + batch_iter = self.generate_batch(batch_samples) + for sample in batch_iter(): + sys.stdout.write(self._gen_str(sample)) + batch_samples = [] + if len(batch_samples) > 0: + batch_iter = self.generate_batch(batch_samples) + for sample in batch_iter(): + sys.stdout.write(self._gen_str(sample)) + + def _gen_str(self, line): + ''' + Further processing the output of the process() function rewritten by + user, outputting data that can be directly read by the datafeed,and + updating proto_info information. + Args: + line(str): the output of the process() function rewritten by user. + Returns: + Return a string data that can be read directly by the datafeed. + ''' + raise NotImplementedError( + "pls use MultiSlotDataGenerator or PairWiseDataGenerator") + + def generate_sample(self, line): + ''' + This function needs to be overridden by the user to process the + original data row into a list or tuple. + Args: + line(str): the original data row + Returns: + Returns the data processed by the user. + The data format is list or tuple: + [(name, [feasign, ...]), ...] + or ((name, [feasign, ...]), ...) + + For example: + [("words", [1926, 08, 17]), ("label", [1])] + or (("words", [1926, 08, 17]), ("label", [1])) + Note: + The type of feasigns must be in int or float. Once the float + element appears in the feasign, the type of that slot will be + processed into a float. + Example: + .. code-block:: python + import paddle.fluid.incubate.data_generator as dg + class MyData(dg.DataGenerator): + def generate_sample(self, line): + def local_iter(): + int_words = [int(x) for x in line.split()] + yield ("words", [int_words]) + return local_iter + ''' + raise NotImplementedError( + "Please rewrite this function to return a list or tuple: " + + "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)") + + def generate_batch(self, samples): + ''' + This function needs to be overridden by the user to process the + generated samples from generate_sample(self, str) function + It is usually used as batch processing when a user wants to + do preprocessing on a batch of samples, e.g. padding according to + the max length of a sample in the batch + Args: + samples(list tuple): generated sample from generate_sample + Returns: + a python generator, the same format as return value of generate_sample + Example: + .. code-block:: python + import paddle.fluid.incubate.data_generator as dg + class MyData(dg.DataGenerator): + def generate_sample(self, line): + def local_iter(): + int_words = [int(x) for x in line.split()] + yield ("words", int_words) + return local_iter + def generate_batch(self, samples): + def local_iter(): + for s in samples: + yield ("words", s[1].extend([s[1][0]])) + mydata = MyData() + mydata.set_batch(128) + ''' + + def local_iter(): + for sample in samples: + yield sample + + return local_iter + + +# TODO: guru4elephant +# add more generalized DataGenerator that can adapt user-defined slot +# for example, [(name, float_list), (name, str_list), (name, int_list)] +class MultiSlotStringDataGenerator(DataGenerator): + def _gen_str(self, line): + ''' + Further processing the output of the process() function rewritten by + user, outputting data that can be directly read by the MultiSlotDataFeed, + and updating proto_info information. + The input line will be in this format: + >>> [(name, [str(feasign), ...]), ...] + >>> or ((name, [str(feasign), ...]), ...) + The output will be in this format: + >>> [ids_num id1 id2 ...] ... + For example, if the input is like this: + >>> [("words", ["1926", "08", "17"]), ("label", ["1"])] + >>> or (("words", ["1926", "08", "17"]), ("label", ["1"])) + the output will be: + >>> 3 1234 2345 3456 1 1 + Args: + line(str): the output of the process() function rewritten by user. + Returns: + Return a string data that can be read directly by the MultiSlotDataFeed. + ''' + if not isinstance(line, list) and not isinstance(line, tuple): + raise ValueError( + "the output of process() must be in list or tuple type" + "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]") + output = "" + for index, item in enumerate(line): + name, elements = item + if output: + output += " " + out_str = [] + out_str.append(str(len(elements))) + out_str.extend(elements) + output += " ".join(out_str) + return output + "\n" + + +class MultiSlotDataGenerator(DataGenerator): + def _gen_str(self, line): + ''' + Further processing the output of the process() function rewritten by + user, outputting data that can be directly read by the MultiSlotDataFeed, + and updating proto_info information. + The input line will be in this format: + >>> [(name, [feasign, ...]), ...] + >>> or ((name, [feasign, ...]), ...) + The output will be in this format: + >>> [ids_num id1 id2 ...] ... + The proto_info will be in this format: + >>> [(name, type), ...] + + For example, if the input is like this: + >>> [("words", [1926, 08, 17]), ("label", [1])] + >>> or (("words", [1926, 08, 17]), ("label", [1])) + the output will be: + >>> 3 1234 2345 3456 1 1 + the proto_info will be: + >>> [("words", "uint64"), ("label", "uint64")] + Args: + line(str): the output of the process() function rewritten by user. + Returns: + Return a string data that can be read directly by the MultiSlotDataFeed. + ''' + if not isinstance(line, list) and not isinstance(line, tuple): + raise ValueError( + "the output of process() must be in list or tuple type" + "Example: [('words', [1926, 08, 17]), ('label', [1])]") + output = "" + + if self._proto_info is None: + self._proto_info = [] + for item in line: + name, elements = item + if not isinstance(name, str): + raise ValueError("name%s must be in str type" % type(name)) + if not isinstance(elements, list): + raise ValueError("elements%s must be in list type" % + type(elements)) + if not elements: + raise ValueError( + "the elements of each field can not be empty, you need padding it in process()." + ) + self._proto_info.append((name, "uint64")) + if output: + output += " " + output += str(len(elements)) + for elem in elements: + if isinstance(elem, float): + self._proto_info[-1] = (name, "float") + elif not isinstance(elem, int) and not isinstance(elem, + long): + raise ValueError( + "the type of element%s must be in int or float" % + type(elem)) + output += " " + str(elem) + else: + if len(line) != len(self._proto_info): + raise ValueError( + "the complete field set of two given line are inconsistent.") + for index, item in enumerate(line): + name, elements = item + if not isinstance(name, str): + raise ValueError("name%s must be in str type" % type(name)) + if not isinstance(elements, list): + raise ValueError("elements%s must be in list type" % + type(elements)) + if not elements: + raise ValueError( + "the elements of each field can not be empty, you need padding it in process()." + ) + if name != self._proto_info[index][0]: + raise ValueError( + "the field name of two given line are not match: require<%s>, get<%s>." + % (self._proto_info[index][0], name)) + if output: + output += " " + output += str(len(elements)) + for elem in elements: + if self._proto_info[index][1] != "float": + if isinstance(elem, float): + self._proto_info[index] = (name, "float") + elif not isinstance(elem, int) and not isinstance(elem, + long): + raise ValueError( + "the type of element%s must be in int or float" + % type(elem)) + output += " " + str(elem) + return output + "\n" diff --git a/python/setup.py.in b/python/setup.py.in index 0e94d02cd6f9b..d9ca3038fb2b7 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -188,6 +188,7 @@ packages=['paddle', 'paddle.fluid.transpiler', 'paddle.fluid.transpiler.details', 'paddle.fluid.incubate', + 'paddle.fluid.incubate.data_generator', 'paddle.fluid.incubate.fleet', 'paddle.fluid.incubate.checkpoint', 'paddle.fluid.incubate.fleet.base', From 7e35ef3a50effe9a2feb481c555fd7b0e6359a9c Mon Sep 17 00:00:00 2001 From: WeiXin Date: Fri, 7 May 2021 09:16:29 +0800 Subject: [PATCH 041/156] [Cherry-Pick] Clear 'BasicEngine' when an exception occurs in the backward. (#32546) (#32615) * clear 'BasicEngine' when an exception occurs in the backward. (#32546) * clear 'BasicEngine' when an exception occurs in the backward. * deal with conflict. * deal with conflict. * forward return any type. (#32661) --- paddle/fluid/imperative/basic_engine.cc | 20 +++-- paddle/fluid/imperative/py_layer_fwd.h | 20 +++-- paddle/fluid/operators/py_layer_op.cc | 6 ++ .../fluid/tests/unittests/test_pylayer_op.py | 79 +++++++++++-------- 4 files changed, 80 insertions(+), 45 deletions(-) diff --git a/paddle/fluid/imperative/basic_engine.cc b/paddle/fluid/imperative/basic_engine.cc index 896918a607106..7bcc3d6c608c9 100644 --- a/paddle/fluid/imperative/basic_engine.cc +++ b/paddle/fluid/imperative/basic_engine.cc @@ -471,12 +471,20 @@ void BasicEngine::Execute() { { VLOG(3) << "Start to execute grad op " << cur_op.Type(); - if (tmp_ins_ptr == nullptr) { - OpBase::Run(cur_op.InnerOp(), bwd_ins, tmp_outs, cur_op.Attrs(), - cur_op.place()); - } else { - OpBase::Run(cur_op.InnerOp(), *tmp_ins_ptr, tmp_outs, cur_op.Attrs(), - cur_op.place()); + try { + if (tmp_ins_ptr == nullptr) { + OpBase::Run(cur_op.InnerOp(), bwd_ins, tmp_outs, cur_op.Attrs(), + cur_op.place()); + } else { + OpBase::Run(cur_op.InnerOp(), *tmp_ins_ptr, tmp_outs, + cur_op.Attrs(), cur_op.place()); + } + } catch (platform::EnforceNotMet& exception) { + Clear(); + throw std::move(exception); + } catch (std::exception& ex) { + Clear(); + PADDLE_THROW(platform::errors::External("%s", ex.what())); } } diff --git a/paddle/fluid/imperative/py_layer_fwd.h b/paddle/fluid/imperative/py_layer_fwd.h index bd132f2576fec..ccfd5b0e2dbfc 100644 --- a/paddle/fluid/imperative/py_layer_fwd.h +++ b/paddle/fluid/imperative/py_layer_fwd.h @@ -115,12 +115,12 @@ py::object PyLayerApply(const platform::Place& place, const py::object& cls, tuple_result[i].cast>(); output_vars.push_back(temp_out); } catch (py::cast_error&) { - PADDLE_THROW(platform::errors::Unimplemented( - "The output of `PyLayer.forward` should be `Tensor`.")); + // Only collect Tensor type in 'kwargs' and pass them to backward. + // Ignore other types of input temporarily. } } else { - PADDLE_THROW(platform::errors::Unimplemented( - "The output of `PyLayer.forward` can not be `None`.")); + // Only collect Tensor type in 'kwargs' and pass them to backward. + // Ignore other types of input temporarily. } } } else { @@ -130,14 +130,18 @@ py::object PyLayerApply(const platform::Place& place, const py::object& cls, result_forward.cast>(); output_vars.push_back(temp_out); } catch (py::cast_error&) { - PADDLE_THROW(platform::errors::Unimplemented( - "The output of `PyLayer.forward` should be `Tensor`.")); + // Only collect Tensor type in 'kwargs' and pass them to backward. + // Ignore other types of input temporarily. } } else { - PADDLE_THROW(platform::errors::Unimplemented( - "The output of `PyLayer.forward` can not be `None`.")); + // Only collect Tensor type in 'kwargs' and pass them to backward. + // Ignore other types of input temporarily. } } + if (output_vars.size() == 0) { + PADDLE_THROW(platform::errors::InvalidArgument( + "At least one output of `PyLayer.forward` is a `Tensor`.")); + } NameVarBaseMap outs = {{"Out", output_vars}}; diff --git a/paddle/fluid/operators/py_layer_op.cc b/paddle/fluid/operators/py_layer_op.cc index 65e10181dcc3d..0090747d1161a 100644 --- a/paddle/fluid/operators/py_layer_op.cc +++ b/paddle/fluid/operators/py_layer_op.cc @@ -86,6 +86,12 @@ void RunPyObject(py::object *py_object, } } } else { + if (1 != outs->size()) { + PADDLE_THROW(platform::errors::InvalidArgument( + "The number of outputs of `PyLayer.backward` should be %d, but " + "received 1.", + outs->size())); + } if ((*outs)[0] != nullptr) { if (Py_None != py_result.ptr()) { try { diff --git a/python/paddle/fluid/tests/unittests/test_pylayer_op.py b/python/paddle/fluid/tests/unittests/test_pylayer_op.py index d329bf570a584..e058115d69199 100644 --- a/python/paddle/fluid/tests/unittests/test_pylayer_op.py +++ b/python/paddle/fluid/tests/unittests/test_pylayer_op.py @@ -30,7 +30,7 @@ def forward(ctx, x1, x2, func1, func2=paddle.square): y1 = func1(x1) y2 = func1(x2) ctx.save_for_backward(y1, y2) - return y1, y2 + return y1, 1, y2, None @staticmethod def backward(ctx, dy1, dy2): @@ -44,7 +44,7 @@ def backward(ctx, dy1, dy2): input1.stop_gradient = False input2.stop_gradient = False z = tanh.apply(input1, input1, paddle.tanh, paddle.square) - z = z[0] + z[1] + z = z[0] + z[2] z.mean().backward() z2 = paddle.tanh(input2) + paddle.tanh(input2) @@ -61,7 +61,7 @@ def forward(ctx, x1, x2, func1, func2=paddle.square): y1 = func1(x1) y2 = func1(x2) ctx.save_for_backward(y1, y2) - return y1, y2 + return 1, None, y1, y2, '' @staticmethod def backward(ctx, dy1, dy2): @@ -79,7 +79,7 @@ def backward(ctx, dy1, dy2): input3.stop_gradient = True input4.stop_gradient = True z = tanh.apply(input1, input3, paddle.tanh, paddle.square) - z = z[0] + z[1] + z = z[2] + z[3] z.mean().backward() z2 = paddle.tanh(input2) + paddle.tanh(input4) @@ -115,6 +115,27 @@ def backward(ctx, dy1): self.assertTrue( np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10) + def test_pylayer_num_output_match(self): + class tanh(PyLayer): + @staticmethod + def forward( + ctx, + x1, + x2, ): + return x1 + x2 + + @staticmethod + def backward(ctx, dy1): + return dy1 + 1 + + input1 = paddle.randn([2, 3]).astype("float64") + input2 = input1.detach().clone() + input1.stop_gradient = False + input2.stop_gradient = False + z = tanh.apply(input1, input2) + with self.assertRaises(ValueError): + z.mean().backward() + def test_pylayer_dtype(self): class tanh(PyLayer): @staticmethod @@ -150,21 +171,21 @@ def backward(ctx, *args): return args input1 = paddle.randn([2, 3]).astype("float64") - with self.assertRaises(NotImplementedError): + with self.assertRaises(ValueError): z = Layer_None1.apply(input1) class Layer_None2(PyLayer): @staticmethod def forward(ctx, *args): - return [None, None] + return [None, args[0]] @staticmethod def backward(ctx, *args): return args input1 = paddle.randn([2, 3]).astype("float64") - with self.assertRaises(NotImplementedError): - z = Layer_None2.apply(input1) + # return None + z = Layer_None2.apply(input1) class Layer_one1(PyLayer): @staticmethod @@ -176,21 +197,22 @@ def backward(ctx, *args): return args input1 = paddle.randn([2, 3]).astype("float64") - with self.assertRaises(NotImplementedError): + # At least one output of `PyLayer.backward` is a `Tensor` + with self.assertRaises(ValueError): z = Layer_one1.apply(input1) class Layer_one2(PyLayer): @staticmethod def forward(ctx, *args): - return [1, 2] + return [1, 2, args[0]] @staticmethod def backward(ctx, *args): return args input1 = paddle.randn([2, 3]).astype("float64") - with self.assertRaises(NotImplementedError): - z = Layer_one2.apply(input1) + # return int + z = Layer_one2.apply(input1) class Layer_no_fw(PyLayer): @staticmethod @@ -234,8 +256,7 @@ def backward(ctx, dy1): z = Layer_bk_none1.apply(input2) with self.assertRaises(ValueError): - with paddle.fluid.dygraph.guard(): - z.sum().backward() + z.sum().backward() class Layer_bk_none2(PyLayer): @staticmethod @@ -249,9 +270,9 @@ def backward(ctx, dy1): input1 = paddle.randn([2, 3]).astype("float64") input1.stop_gradient = False z = Layer_bk_none2.apply(input1, input1) + with self.assertRaises(ValueError): - with paddle.fluid.dygraph.guard(): - z.mean().backward() + z.mean().backward() class Layer_bk_one1(PyLayer): @staticmethod @@ -265,9 +286,9 @@ def backward(ctx, dy): input1 = paddle.randn([2, 3]).astype("float64") input1.stop_gradient = False z = Layer_bk_one1.apply(input1) + with self.assertRaises(ValueError): - with paddle.fluid.dygraph.guard(): - z.mean().backward() + z.mean().backward() class Layer_bk_one2(PyLayer): @staticmethod @@ -280,11 +301,11 @@ def backward(ctx, *args): input1 = paddle.randn([2, 3]).astype("float64") input1.stop_gradient = False + y = Layer_bk_one2.apply(input1, input1) z = y[0] + y[1] with self.assertRaises(ValueError): - with paddle.fluid.dygraph.guard(): - z.mean().backward() + z.mean().backward() class Layer_no_bk(PyLayer): @staticmethod @@ -295,10 +316,9 @@ def forward(ctx, x): input1.stop_gradient = False z = Layer_no_bk.apply(input1) - with self.assertRaises(NotImplementedError): - with paddle.fluid.dygraph.guard(): - z = z[0] + z[1] - z.mean().backward() + with self.assertRaises(OSError): + z = z[0] + z[1] + z.mean().backward() class Layer_bk_match(PyLayer): @staticmethod @@ -313,9 +333,8 @@ def backward(ctx, dy1, dy2): input1.stop_gradient = False z = Layer_bk_match.apply(input1) with self.assertRaises(ValueError): - with paddle.fluid.dygraph.guard(): - z = z[0] + z[1] - z.mean().backward() + z = z[0] + z[1] + z.mean().backward() def test_pylayer_bk_return_none(self): class Layer_bk_none1(PyLayer): @@ -334,8 +353,7 @@ def backward(ctx, dy): z = Layer_bk_none1.apply(input1, input2) with self.assertRaises(ValueError): - with paddle.fluid.dygraph.guard(): - z.mean().backward() + z.mean().backward() class Layer_bk_none2(PyLayer): @staticmethod @@ -353,8 +371,7 @@ def backward(ctx, *args): z = Layer_bk_none2.apply(input1, input2) z = z[0] + z[1] with self.assertRaises(ValueError): - with paddle.fluid.dygraph.guard(): - z.mean().backward() + z.mean().backward() def test_pylayer_inplace(self): class cus_tanh(PyLayer): From c67a5d98d88abd729c928b05530131052e21bfc9 Mon Sep 17 00:00:00 2001 From: WeiXin Date: Fri, 7 May 2021 09:17:25 +0800 Subject: [PATCH 042/156] pylayer_op:release context after compute. (#32707) (#32744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复了py_layer_op由于没有析构PyLayerContext造成内存(显存)泄露的问题。 原始pr：#32707 --- paddle/fluid/imperative/py_layer_fwd.h | 5 +++-- paddle/fluid/operators/py_layer_op.cc | 9 ++++++--- paddle/fluid/operators/py_layer_op.h | 11 +++++++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/imperative/py_layer_fwd.h b/paddle/fluid/imperative/py_layer_fwd.h index ccfd5b0e2dbfc..de5f9d75e9173 100644 --- a/paddle/fluid/imperative/py_layer_fwd.h +++ b/paddle/fluid/imperative/py_layer_fwd.h @@ -63,15 +63,16 @@ std::shared_ptr CreateGradOpNode( } } -py::object PyLayerApply(const platform::Place& place, const py::object& cls, +py::object PyLayerApply(const platform::Place& place, const py::handle& cls, const py::args args, const py::kwargs kwargs) { + py::gil_scoped_acquire guard; auto bk_function = cls.attr("_backward_function"); auto context = bk_function(); auto forward = cls.attr("forward"); auto result_forward = forward(context, *args, **kwargs); std::shared_ptr py_layer_ctx = - std::make_shared(context.release().ptr()); + std::make_shared(context.ptr()); // make inputs to varbase std::vector> input_vars; // process args,`input_vars` only collect `imperative::VarBase` diff --git a/paddle/fluid/operators/py_layer_op.cc b/paddle/fluid/operators/py_layer_op.cc index 0090747d1161a..f91496eeab142 100644 --- a/paddle/fluid/operators/py_layer_op.cc +++ b/paddle/fluid/operators/py_layer_op.cc @@ -157,9 +157,12 @@ class PyLayerOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { auto &op_ = ctx.GetOp(); - auto pylayer_op = dynamic_cast(&op_); - if (pylayer_op) { - auto py_layer_context = pylayer_op->GetPyLayerContext(); + auto const_pylayer_op = dynamic_cast(&op_); + if (const_pylayer_op) { + auto pylayer_op = const_cast(const_pylayer_op); + + // Release contex after executing the compute + auto py_layer_context = pylayer_op->ReleasePyLayerContext(); py::object bk_ctx(py::handle(py_layer_context->GetMutableCtx()), true); auto &input_vars = ctx.MultiInputVar("X"); auto output_vars = ctx.MultiOutputVar("Out"); diff --git a/paddle/fluid/operators/py_layer_op.h b/paddle/fluid/operators/py_layer_op.h index 133435aa84d71..d80faab90b223 100644 --- a/paddle/fluid/operators/py_layer_op.h +++ b/paddle/fluid/operators/py_layer_op.h @@ -34,6 +34,10 @@ class PyLayerContext { PyLayerContext() = delete; PyObject* GetMutableCtx() { return context_; } + ~PyLayerContext() { + py::gil_scoped_acquire guard; + Py_XDECREF(context_); + } private: PyObject* context_; @@ -58,8 +62,11 @@ class PyLayerOp : public framework::OperatorWithKernel { void SetPyLayerContext(const std::shared_ptr& py_context) { py_context_ = py_context; } - const std::shared_ptr& GetPyLayerContext() const { - return py_context_; + std::shared_ptr ReleasePyLayerContext() { + auto temp = py_context_; + py_context_.reset(); + VLOG(3) << "`py_context_` in the PyLayerOp is released."; + return temp; } private: From ce27821dc28153e671f7e4086fd4d0932186bdef Mon Sep 17 00:00:00 2001 From: xiemoyuan <71377852+xiemoyuan@users.noreply.github.com> Date: Fri, 7 May 2021 12:10:47 +0800 Subject: [PATCH 043/156] [2.1 API] Enable printing deprecated warning info. (#32712) (#32756) * Add deprecated warning info. * Add unittest for deprecated decorator. * Add warning info for tensor.grad --- python/paddle/dataset/cifar.py | 5 +++ python/paddle/dataset/conll05.py | 4 ++ python/paddle/dataset/flowers.py | 3 ++ python/paddle/dataset/imdb.py | 5 +++ python/paddle/dataset/imikolov.py | 3 ++ python/paddle/dataset/mnist.py | 3 ++ python/paddle/dataset/movielens.py | 9 ++++ python/paddle/dataset/uci_housing.py | 4 ++ python/paddle/dataset/voc2012.py | 3 ++ python/paddle/dataset/wmt14.py | 5 +++ python/paddle/dataset/wmt16.py | 5 +++ .../fluid/dygraph/varbase_patch_methods.py | 6 ++- .../unittests/test_deprecated_decorator.py | 41 +++++++++++++++++++ python/paddle/nn/__init__.py | 9 ++-- python/paddle/nn/functional/loss.py | 8 +++- python/paddle/utils/deprecated.py | 31 ++++++++++---- 16 files changed, 130 insertions(+), 14 deletions(-) diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index e3d239e2cdf45..9a9f9018e4216 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -79,6 +79,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar100", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train100(): """ @@ -98,6 +99,7 @@ def train100(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar100", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test100(): """ @@ -117,6 +119,7 @@ def test100(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train10(cycle=False): """ @@ -139,6 +142,7 @@ def train10(cycle=False): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test10(cycle=False): """ @@ -161,6 +165,7 @@ def test10(cycle=False): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5) diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index 65cf04f05b7f0..f09163ea424b0 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -206,6 +206,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Conll05st", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def get_dict(): """ @@ -223,6 +224,7 @@ def get_dict(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Conll05st", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def get_embedding(): """ @@ -234,6 +236,7 @@ def get_embedding(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Conll05st", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(): """ @@ -258,6 +261,7 @@ def test(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Conll05st", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5) diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 3b437a1f07440..2f38c563136d3 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -156,6 +156,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Flowers", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False): ''' @@ -189,6 +190,7 @@ def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Flowers", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False): ''' @@ -222,6 +224,7 @@ def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Flowers", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def valid(mapper=test_mapper, buffered_size=1024, use_xmap=True): ''' diff --git a/python/paddle/dataset/imdb.py b/python/paddle/dataset/imdb.py index 9a6c8e837ed46..961d238b0ad41 100644 --- a/python/paddle/dataset/imdb.py +++ b/python/paddle/dataset/imdb.py @@ -80,6 +80,7 @@ def build_dict(pattern, cutoff): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def reader_creator(pos_pattern, neg_pattern, word_idx): UNK = word_idx[''] @@ -102,6 +103,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(word_idx): """ @@ -123,6 +125,7 @@ def train(word_idx): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(word_idx): """ @@ -144,6 +147,7 @@ def test(word_idx): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def word_dict(): """ @@ -159,6 +163,7 @@ def word_dict(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(URL, 'imdb', MD5) diff --git a/python/paddle/dataset/imikolov.py b/python/paddle/dataset/imikolov.py index 7a4efe27aa961..85fe011fa143a 100644 --- a/python/paddle/dataset/imikolov.py +++ b/python/paddle/dataset/imikolov.py @@ -115,6 +115,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imikolov", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(word_idx, n, data_type=DataType.NGRAM): """ @@ -139,6 +140,7 @@ def train(word_idx, n, data_type=DataType.NGRAM): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imikolov", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(word_idx, n, data_type=DataType.NGRAM): """ @@ -163,6 +165,7 @@ def test(word_idx, n, data_type=DataType.NGRAM): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imikolov", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(URL, "imikolov", MD5) diff --git a/python/paddle/dataset/mnist.py b/python/paddle/dataset/mnist.py index e4f724bd66d13..02cdd30708392 100644 --- a/python/paddle/dataset/mnist.py +++ b/python/paddle/dataset/mnist.py @@ -93,6 +93,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.MNIST", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(): """ @@ -114,6 +115,7 @@ def train(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.MNIST", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(): """ @@ -134,6 +136,7 @@ def test(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.MNIST", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5) diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index 862ac586bc964..9af06e088ca87 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -168,6 +168,7 @@ def __reader__(rand_seed=0, test_ratio=0.1, is_test=False): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def __reader_creator__(**kwargs): return lambda: __reader__(**kwargs) @@ -180,6 +181,7 @@ def __reader_creator__(**kwargs): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def get_movie_title_dict(): """ @@ -199,6 +201,7 @@ def __max_index_info__(a, b): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def max_movie_id(): """ @@ -211,6 +214,7 @@ def max_movie_id(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def max_user_id(): """ @@ -230,6 +234,7 @@ def __max_job_id_impl__(a, b): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def max_job_id(): """ @@ -243,6 +248,7 @@ def max_job_id(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def movie_categories(): """ @@ -255,6 +261,7 @@ def movie_categories(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def user_info(): """ @@ -267,6 +274,7 @@ def user_info(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def movie_info(): """ @@ -288,6 +296,7 @@ def unittest(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(URL, "movielens", MD5) diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 0ac65f0fda46b..dea2dfc8c9818 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -87,6 +87,7 @@ def load_data(filename, feature_num=14, ratio=0.8): @deprecated( since="2.0.0", update_to="paddle.text.datasets.UCIHousing", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(): """ @@ -111,6 +112,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.UCIHousing", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(): """ @@ -146,6 +148,7 @@ def fluid_model(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.UCIHousing", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def predict_reader(): """ @@ -162,6 +165,7 @@ def predict_reader(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.UCIHousing", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(URL, 'uci_housing', MD5) diff --git a/python/paddle/dataset/voc2012.py b/python/paddle/dataset/voc2012.py index 5784e739b418e..1ab91db2cc36d 100644 --- a/python/paddle/dataset/voc2012.py +++ b/python/paddle/dataset/voc2012.py @@ -69,6 +69,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.VOC2012", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(): """ @@ -80,6 +81,7 @@ def train(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.VOC2012", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(): """ @@ -91,6 +93,7 @@ def test(): @deprecated( since="2.0.0", update_to="paddle.vision.datasets.VOC2012", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def val(): """ diff --git a/python/paddle/dataset/wmt14.py b/python/paddle/dataset/wmt14.py index c842ceaa09133..9f8abb2c4bfe9 100644 --- a/python/paddle/dataset/wmt14.py +++ b/python/paddle/dataset/wmt14.py @@ -114,6 +114,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(dict_size): """ @@ -134,6 +135,7 @@ def train(dict_size): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(dict_size): """ @@ -154,6 +156,7 @@ def test(dict_size): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def gen(dict_size): return reader_creator( @@ -164,6 +167,7 @@ def gen(dict_size): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def get_dict(dict_size, reverse=True): # if reverse = False, return dict = {'a':'001', 'b':'002', ...} @@ -179,6 +183,7 @@ def get_dict(dict_size, reverse=True): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN) diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index 320ef139f7700..f313da98f0abc 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -142,6 +142,7 @@ def reader(): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def train(src_dict_size, trg_dict_size, src_lang="en"): """ @@ -195,6 +196,7 @@ def train(src_dict_size, trg_dict_size, src_lang="en"): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def test(src_dict_size, trg_dict_size, src_lang="en"): """ @@ -248,6 +250,7 @@ def test(src_dict_size, trg_dict_size, src_lang="en"): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def validation(src_dict_size, trg_dict_size, src_lang="en"): """ @@ -299,6 +302,7 @@ def validation(src_dict_size, trg_dict_size, src_lang="en"): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def get_dict(lang, dict_size, reverse=False): """ @@ -333,6 +337,7 @@ def get_dict(lang, dict_size, reverse=False): @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", + level=1, reason="Please use new dataset API which supports paddle.io.DataLoader") def fetch(): """download the entire dataset. diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index bb84b2ca9705c..37900b7880a35 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -241,7 +241,8 @@ def backward(self, grad_tensor=None, retain_graph=False): @framework.dygraph_only @deprecated( since="2.1.0", - reason="Please use x.grad, which returns the tensor value of the gradient." + level=1, + reason="Please use tensor.grad, which returns the tensor value of the gradient." ) def gradient(self): """ @@ -367,6 +368,9 @@ def grad(self): # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=False, [500.]) """ + msg = "tensor.grad will return the tensor value of the gradient." + warning_msg = "\033[93m\nWarning:\n%s \033[0m" % (msg) + warnings.warn(warning_msg) return self._grad_ivar() def clear_grad(self): diff --git a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py index 97b6594eb3825..7dc5dc70618e6 100755 --- a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py +++ b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py @@ -21,6 +21,8 @@ import unittest import paddle.fluid.core as core import sys +import warnings +import paddle.utils.deprecated as deprecated LOWEST_WARNING_POSTION = 3 ERROR_WARNING_POSTION = sys.maxsize @@ -149,6 +151,45 @@ def test_ops_elementwise_mul(self): # testting self.assertGreater(expected, captured) + def test_tensor_gradient(self): + paddle.__version__ = '2.1.0' + + x = paddle.to_tensor(5., stop_gradient=False) + y = paddle.pow(x, 4.0) + y.backward() + + with warnings.catch_warnings(record=True) as w: + grad = x.gradient() + assert ( + 'API "paddle.fluid.dygraph.varbase_patch_methods.gradient" is ' + 'deprecated since 2.1.0') in str(w[-1].message) + + def test_softmax_with_cross_entropy(self): + paddle.__version__ = '2.0.0' + + data = np.random.rand(128).astype("float32") + label = np.random.rand(1).astype("int64") + data = paddle.to_tensor(data) + label = paddle.to_tensor(label) + linear = paddle.nn.Linear(128, 100) + x = linear(data) + + with warnings.catch_warnings(record=True) as w: + out = paddle.nn.functional.softmax_with_cross_entropy( + logits=x, label=label) + assert ( + 'API "paddle.nn.functional.loss.softmax_with_cross_entropy" is ' + 'deprecated since 2.0.0') in str(w[-1].message) + + def test_deprecated_error(self): + paddle.__version__ = '2.1.0' + + @deprecated(since="2.1.0", level=2) + def deprecated_error_func(): + pass + + self.assertRaises(RuntimeError, deprecated_error_func) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 817fd50118199..4e4669892b0f0 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -146,7 +146,8 @@ @deprecated( since="2.0.0", update_to="paddle.nn.funcitional.diag_embed", - reason="diag_embed in paddle.nn will removed in future") + level=1, + reason="diag_embed in paddle.nn will be removed in future") def diag_embed(*args): ''' alias name of paddle.nn.functional.diag_embed @@ -157,7 +158,8 @@ def diag_embed(*args): @deprecated( since="2.0.0", update_to="paddle.nn.utils.remove_weight_norm", - reason="remove_weight_norm in paddle.nn will removed in future") + level=1, + reason="remove_weight_norm in paddle.nn will be removed in future") def remove_weight_norm(*args): ''' alias name of paddle.nn.utils.remove_weight_norm @@ -168,7 +170,8 @@ def remove_weight_norm(*args): @deprecated( since="2.0.0", update_to="paddle.nn.utils.weight_norm", - reason="weight_norm in paddle.nn will removed in future") + level=1, + reason="weight_norm in paddle.nn will be removed in future") def weight_norm(*args): ''' alias name of paddle.nn.utils.weight_norm diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index b89da3d82e379..aa0bd8a8c5e3d 100755 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -1096,7 +1096,13 @@ def ctc_loss(log_probs, return loss_out -@deprecated(since="2.0.0", update_to="paddle.nn.functional.cross_entropy") +@deprecated( + since="2.0.0", + update_to="paddle.nn.functional.cross_entropy", + level=1, + reason=( + 'Please notice that behavior of "paddle.nn.functional.softmax_with_cross_entropy" ' + 'and "paddle.nn.functional.cross_entropy" is different.')) def softmax_with_cross_entropy(logits, label, soft_label=False, diff --git a/python/paddle/utils/deprecated.py b/python/paddle/utils/deprecated.py index 5390dea69fe7d..e3839d9767d21 100755 --- a/python/paddle/utils/deprecated.py +++ b/python/paddle/utils/deprecated.py @@ -30,7 +30,7 @@ warnings.simplefilter('default', DeprecationWarning) -def deprecated(update_to="", since="", reason=""): +def deprecated(update_to="", since="", reason="", level=0): """Decorate a function to signify its deprecation. This function wraps a method that will soon be removed and does two things: @@ -39,9 +39,14 @@ def deprecated(update_to="", since="", reason=""): - Raises a :class:`~exceptions.DeprecatedWarning` when old API is called. Args: - since(str): The version at which the decorated method is considered deprecated. - update_to(str): The new API users should use. - reason(str): The reason why the API is deprecated. + since(str, optional): The version at which the decorated method is considered deprecated. + update_to(str, optional): The new API users should use. + reason(str, optional): The reason why the API is deprecated. + level(int, optional): The deprecated warning log level. It must be + an Integer and must be one of 0, 1, 2. + If `level == 0`, the warning message will not be showed. + If `level == 1`, the warning message will be showed normally. + If `level == 2`, it will raise `RuntimeError`. Returns: decorator: decorated function or class. @@ -54,6 +59,9 @@ def decorator(func): assert isinstance(update_to, str), 'type of "update_to" must be str.' assert isinstance(since, str), 'type of "since" must be str.' assert isinstance(reason, str), 'type of "reason" must be str.' + assert isinstance(level, int) and level >= 0 and level < 3, ( + 'type of "level" must be int and must be one of 0, 1, 2. But ' + 'received: {}.'.format(level)) _since = since.strip() _update_to = update_to.strip() @@ -71,12 +79,12 @@ def decorator(func): update_to) msg += ' Please use "{}" instead.'.format(_update_to) if len(_reason) > 0: - msg += "\n reason: {}".format(_reason) + msg += "\nreason: {}".format(_reason) if func.__doc__: func.__doc__ = ('\n\nWarning: ' + msg + '\n') + func.__doc__ - # TODO(Joejiong) Early returning the wrapper function, currently we disable the warning wrapper, - # because the 2.0beta APIs are still under development, we will restore the warning functionality when 2.0 rc APIs become stable. - return func + + if level == 0: + return func @functools.wraps(func) def wrapper(*args, **kwargs): @@ -85,7 +93,12 @@ def wrapper(*args, **kwargs): 2. since version is empty, in this case, API is deprecated in all versions. 3. current version is newer than since version. """ - warningmsg = "\033[93mWarning %s \033[0m" % (msg) + + if level == 2: + raise RuntimeError('API "{}.{}" has been deprecated.'.format( + func.__module__, func.__name__)) + + warningmsg = "\033[93m\nWarning:\n%s \033[0m" % (msg) v_current = [int(i) for i in paddle.__version__.split(".")] v_current += [0] * (4 - len(v_current)) v_since = [int(i) for i in _since.split(".")] From 5fdd85ba4b148d498c1ff9b1acfbb9d4a70ac241 Mon Sep 17 00:00:00 2001 From: lilong12 Date: Fri, 7 May 2021 14:07:03 +0800 Subject: [PATCH 044/156] bug fix, test=develop (#32753) --- python/paddle/distributed/fleet/base/topology.py | 2 +- .../distributed/fleet/meta_parallel/pipeline_parallel.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py index 8f38ba447fcb3..470a4d83aac3f 100644 --- a/python/paddle/distributed/fleet/base/topology.py +++ b/python/paddle/distributed/fleet/base/topology.py @@ -147,7 +147,7 @@ def __init__(self, topology): debug_str = "HybridParallelInfo: rank_id: %d, dp_degree: %d, " \ "mp_degree: %d, pp_degree: %d" % (self.global_rank, self._dp_degree, self._mp_degree,self._pp_degree) - debug_str += "dp_group: %s, mp_group: %s, pp_group: %s, check/clip group: %s" % ( + debug_str += ", dp_group: %s, mp_group: %s, pp_group: %s, check/clip group: %s" % ( self._dp_group, self._mp_group, self._pp_group, self._check_group) logger.info(debug_str) diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 11180054afbfc..8fb29a4485df0 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -125,9 +125,9 @@ def _forward(self, cache_id): self._recv_activations(cache_id) if isinstance(self.caches['inputs'][cache_id], tuple): - inputs = tuple(t.clone() for t in self.caches['inputs'][cache_id]) + inputs = tuple(t for t in self.caches['inputs'][cache_id]) else: - inputs = self.caches['inputs'][cache_id].clone() + inputs = self.caches['inputs'][cache_id] self._clear_grads(inputs) outputs = self._layers.forward(inputs) From 70e0e3d53f7375bd17fb8b9dd6ba0802990800ae Mon Sep 17 00:00:00 2001 From: lidanqing Date: Fri, 7 May 2021 08:18:28 +0200 Subject: [PATCH 045/156] [cherry-pick] Mechanism that converts startup_program initializers to BF16 (#32720) (#32764) * Add casting initializers for bf16 training * Changes after review * Correct test and add comment Co-authored-by: joanna.wozna.intel --- .../contrib/mixed_precision/bf16/amp_lists.py | 3 ++ .../contrib/mixed_precision/bf16/amp_utils.py | 51 ++++++++++++++++++- .../contrib/mixed_precision/bf16/decorator.py | 11 ++-- .../contrib/mixed_precision/fp16_utils.py | 30 +++++++---- .../fluid/contrib/tests/test_bf16_utils.py | 23 +++++++++ .../contrib/tests/test_model_cast_to_bf16.py | 28 ++++++---- python/paddle/fluid/layers/tensor.py | 10 ++-- .../fluid/tests/book/test_fit_a_line.py | 3 +- .../fluid/tests/book/test_word2vec_book.py | 2 +- 9 files changed, 131 insertions(+), 30 deletions(-) diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py index 1cf54aa0838ab..3a4dc8ed9afcc 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py @@ -49,6 +49,7 @@ def __init__(self, self.bf16_list = copy.copy(bf16_list) self.fp32_list = copy.copy(fp32_list) self.gray_list = copy.copy(gray_list) + self.bf16_initializer_list = copy.copy(bf16_initializer_list) self.unsupported_list = copy.copy(unsupported_list) self.fp32_varnames = copy.copy(custom_fp32_varnames) self._update_list() @@ -79,6 +80,8 @@ def _update_list(self): self.unsupported_list.add(op_name) +bf16_initializer_list = {'fill_constant', 'uniform_random'} + # always bf16 bf16_list = {'elementwise_add', } diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py index 038479098a623..4551947e0fad2 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py @@ -232,7 +232,52 @@ def bf16_guard(): yield -def cast_model_to_bf16(program, amp_lists=None, use_bf16_guard=True): +def are_post_ops_bf16(post_ops, keep_fp32_ops): + for post_op in post_ops: + for op in post_op: + if op.type in keep_fp32_ops: + return False + return True + + +def cast_initializers_to_bf16(startup_prog, + amp_lists, + block, + all_ops, + keep_fp32_ops, + to_bf16_var_names=None): + prepend_ops = startup_prog.global_block().ops + for op in prepend_ops: + if str(op.type) in amp_lists.bf16_initializer_list: + change_op = True + op_post_ops = [] + op_out_vars = [] + for out_name in op.output_names: + for out_var_name in op.output(out_name): + out_var = block.var(out_var_name) + post_op = find_true_post_op(all_ops, op, out_var_name, True) + + if out_var is None or out_var.type not in _valid_types: + change_op = False + break + op_post_ops.append(post_op) + op_out_vars.append(out_var) + + if change_op and are_post_ops_bf16(op_post_ops, keep_fp32_ops): + for out_var in op_out_vars: + if out_var.dtype == core.VarDesc.VarType.FP32: + out_var.desc.set_dtype(core.VarDesc.VarType.BF16) + if to_bf16_var_names is not None and out_var.name in to_bf16_var_names: + to_bf16_var_names.remove(out_var.name) + if op.has_attr('dtype') and op.attr( + 'dtype') == core.VarDesc.VarType.FP32: + op._set_attr('dtype', core.VarDesc.VarType.BF16) + + +def cast_model_to_bf16(program, + startup_prog=None, + amp_lists=None, + use_bf16_guard=True): """ Traverse all ops in the whole model and set their inputs and outputs to the bf16 data type. This function will do some special processing for @@ -329,6 +374,10 @@ def cast_model_to_bf16(program, amp_lists=None, use_bf16_guard=True): if op.has_attr('mkldnn_data_type'): op._set_attr('mkldnn_data_type', 'bfloat16') + if startup_prog is not None: + cast_initializers_to_bf16(startup_prog, amp_lists, global_block, + ops, keep_fp32_ops, to_bf16_var_names) + # process ops in keep_fp32_ops op_var_rename_map = [ collections.OrderedDict() for _ in range(len(program.blocks)) diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py index 86b5a5df75db0..32c8a1c3544c2 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py @@ -94,7 +94,8 @@ def backward(self, if self._use_pure_bf16: self._to_bf16_var_names = cast_model_to_bf16( - self._train_program, self._amp_lists, self._use_bf16_guard) + self._train_program, startup_program, self._amp_lists, + self._use_bf16_guard) else: rewrite_program_bf16(self._train_program, self._amp_lists) @@ -168,10 +169,12 @@ def run_example_code(): self._to_bf16_var_names) if test_program is not None: if self._use_pure_bf16: - cast_model_to_bf16(test_program, self._amp_lists, - self._use_bf16_guard) + cast_model_to_bf16( + test_program, + amp_lists=self._amp_lists, + use_bf16_guard=self._use_bf16_guard) elif use_bf16_test: - rewrite_program_bf16(test_program, self._amp_lists) + rewrite_program_bf16(test_program, amp_lists=self._amp_lists) def apply_gradients(self, params_grads): """ diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py index 65b62e7e5ab55..16dfb2bd50c14 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py @@ -157,7 +157,8 @@ def _insert_cast_post_op(block, op, idx, src_dtype, dest_dtype, target_name, return num_cast_ops assert target_var.dtype == src_dtype, \ - "The real dtype({}) is not equal to the src dtype({})".format(_dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype)) + "The real dtype({}) is not equal to the src dtype({})".format( + _dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype)) cast_name = target_var.name + '.cast_' + _dtype_to_str(dest_dtype) cast_var = block.vars.get(cast_name) @@ -209,7 +210,7 @@ def find_true_prev_op(ops, cur_op, var_name): return None -def find_true_post_op(ops, cur_op, var_name): +def find_true_post_op(ops, cur_op, var_name, search_all=False): """ if there are post ops, return them, if there is no post op, return None instead. @@ -217,11 +218,22 @@ def find_true_post_op(ops, cur_op, var_name): ops (list): A list of ops. cur_op (Operator): Current operator which has var_name variable. var_name (string): Variable name. + search_all (bool): The type of operator search. Use if \"cur_op\" is not in the \"ops\" set. """ post_op = [] - for idx, op in enumerate(ops): - if op == cur_op: - break + if search_all: + """ + \"cur_op\" do not have to be in list of \"ops\". E.g. \"cur_op\" can come + from startup_prog block and \"ops\" list from main_prog block. + By setting idx to -1, we'll start looking for post-ops from the top of the list. + If search_all is False, assume that \"cur_op\" is in \"ops\" list, + so to reduce the time of search we can start iterating from \"cur_op\" idx. + """ + idx = -1 + else: + for idx, op in enumerate(ops): + if op == cur_op: + break for i in range(idx + 1, len(ops)): op = ops[i] @@ -270,7 +282,7 @@ def _need_keep_fp32(op, unsupported_op_list, use_fp16_guard): if use_fp16_guard: if op.has_attr("op_namescope") and \ - (_fp16_guard_pattern in op.attr("op_namescope")): + (_fp16_guard_pattern in op.attr("op_namescope")): # op in fp16 guard return False else: @@ -496,8 +508,8 @@ def rewrite_program(main_prog, amp_lists): black_op_set = set() for op in ops: - # NOTE(zhiqiu): 'create_py_reader' and 'read' is used in non-iterable DataLoder, - # we don't need to handle reader op and the input of 'create_py_reader' is not + # NOTE(zhiqiu): 'create_py_reader' and 'read' is used in non-iterable DataLoder, + # we don't need to handle reader op and the input of 'create_py_reader' is not # in block, which may result in errors. # See GeneratorLoader._init_non_iterable() for details. if op.type == 'create_py_reader' or op.type == 'read': @@ -612,7 +624,7 @@ def update_role_var_grad(main_prog, params_grads): raise ValueError("The cast op {0}'s output should not be" "used by a non-optimize op, however, it" "is used by {1}".format(op, post_ops[0])) - #add new op in the python and cpp at the same time + # add new op in the python and cpp at the same time new_op_desc = block.desc.append_op() new_op_desc.copy_from(op.desc) new_op = framework.Operator( diff --git a/python/paddle/fluid/contrib/tests/test_bf16_utils.py b/python/paddle/fluid/contrib/tests/test_bf16_utils.py index 2969b7ea11d21..41aa5e5412df5 100644 --- a/python/paddle/fluid/contrib/tests/test_bf16_utils.py +++ b/python/paddle/fluid/contrib/tests/test_bf16_utils.py @@ -139,6 +139,29 @@ def test_find_true_post_op(self): res = amp.bf16.amp_utils.find_true_post_op(block.ops, op1, "Y") assert (res == [op2]) + def test_find_true_post_op_with_search_all(self): + program = fluid.Program() + block = program.current_block() + startup_block = fluid.default_startup_program().global_block() + + var1 = block.create_var(name="X", shape=[3], dtype='float32') + var2 = block.create_var(name="Y", shape=[3], dtype='float32') + inititializer_op = startup_block._prepend_op( + type="fill_constant", + outputs={"Out": var1}, + attrs={"shape": var1.shape, + "dtype": var1.dtype, + "value": 1.0}) + + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]}) + result = amp.bf16.amp_utils.find_true_post_op( + block.ops, inititializer_op, "X", search_all=False) + assert (len(result) == 0) + result = amp.bf16.amp_utils.find_true_post_op( + block.ops, inititializer_op, "X", search_all=True) + assert (result == [op1]) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py index af2c42d6b85ea..470073543c3be 100644 --- a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py +++ b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py @@ -53,19 +53,27 @@ def scope_prog_guard(self): with fluid.program_guard(prog, startup_prog): yield - def get_static_graph_result(self, feed, fetch_list, amp_fun, - with_lod=False): + def get_static_graph_result(self, + feed, + fetch_list, + amp_fun, + with_lod=False, + startup_prog=None): exe = fluid.Executor(core.CPUPlace()) - exe.run(fluid.default_startup_program()) + exe.run(fluid.default_startup_program() + if startup_prog is None else startup_prog) prog = fluid.default_main_program() if amp_fun is not None: - amp_fun(prog) + if startup_prog is not None: + amp_fun(prog, startup_prog) + else: + amp_fun(prog) return exe.run(prog, feed=feed, fetch_list=fetch_list, return_numpy=(not with_lod)) - def _graph_common(self, _amp_fun): + def _graph_common(self, _amp_fun, startup_prog=None): size = 3 n = np.ones([size, size], dtype='float32') * 3.2 nn = np.ones([size, size], dtype='float32') * -2.7 @@ -122,7 +130,8 @@ def _graph_common(self, _amp_fun): self.get_static_graph_result( feed={'t': n, 'tt': nn}, fetch_list=[ret], - amp_fun=_amp_fun + amp_fun=_amp_fun, + startup_prog=startup_prog ) self.assertTrue( static_ret_bf16, np.ones( @@ -132,16 +141,17 @@ def test_graph_rewrite(self): self._graph_common(lambda prog: amp.bf16.rewrite_program_bf16( prog, amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_varnames={'elementwise_add_0.tmp_0'}), + custom_fp32_varnames={'elementwise_add_0.tmp_0'}) )) def test_graph_cast(self): - self._graph_common(lambda prog: amp.bf16.cast_model_to_bf16( + self._graph_common(lambda prog, startup_prog: amp.bf16.cast_model_to_bf16( prog, + startup_prog, amp.bf16.AutoMixedPrecisionListsBF16( custom_fp32_list={'elementwise_mul'}), use_bf16_guard=True - )) + ), startup_prog=fluid.default_startup_program()) if __name__ == '__main__': diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 7dcce5efcfc65..c0c07f593a3ed 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -231,13 +231,13 @@ def cast(x, dtype): out = core.ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) return out - check_variable_and_dtype( - x, 'x', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], - 'cast') + check_variable_and_dtype(x, 'x', [ + 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'uint8', + 'uint16' + ], 'cast') check_dtype(dtype, 'dtype', [ 'bool', 'float16', 'float32', 'float64', 'int8', 'int32', 'int64', - 'uint8' + 'uint8', 'uint16' ], 'cast') helper = LayerHelper('cast', **locals()) diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index 1172ae0f0ea42..12952462270f0 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -56,7 +56,8 @@ def train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16): amp_lists=amp.bf16.AutoMixedPrecisionListsBF16(), use_bf16_guard=False, use_pure_bf16=pure_bf16) - sgd_optimizer.minimize(avg_cost) + sgd_optimizer.minimize( + avg_cost, startup_program=fluid.default_startup_program()) BATCH_SIZE = 20 diff --git a/python/paddle/fluid/tests/book/test_word2vec_book.py b/python/paddle/fluid/tests/book/test_word2vec_book.py index f16592a55cf8a..650ccc0776a50 100644 --- a/python/paddle/fluid/tests/book/test_word2vec_book.py +++ b/python/paddle/fluid/tests/book/test_word2vec_book.py @@ -115,7 +115,7 @@ def __network__(words): use_bf16_guard=False, use_pure_bf16=pure_bf16) - sgd_optimizer.minimize(avg_cost) + sgd_optimizer.minimize(avg_cost, fluid.default_startup_program()) train_reader = paddle.batch( paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE) From 3ba8c48a161d4183e2791b6fb207ae6640780a25 Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Fri, 7 May 2021 15:42:47 +0800 Subject: [PATCH 046/156] [CHERRY-PICK2.1]Remove paddle_custom_op dynamic libraries, and link to FLUID_CORE on windows (#32583) (#32769) * Remove paddle_custom_op dynamic libraries, change link to FLUID_CORE on windows, and check copy_to * fix CI --- paddle/fluid/framework/CMakeLists.txt | 33 --- paddle/scripts/paddle_build.bat | 112 +++++---- python/CMakeLists.txt | 17 +- python/paddle/check_import_scipy.py | 2 +- python/paddle/fluid/core.py | 20 +- .../fluid/tests/custom_op/CMakeLists.txt | 5 +- .../fluid/tests/custom_op/custom_relu_op.cu | 6 +- .../fluid/tests/custom_op/test_check_abi.py | 31 ++- .../custom_op/test_custom_relu_op_jit.py | 10 +- .../utils/cpp_extension/cpp_extension.py | 18 +- .../utils/cpp_extension/extension_utils.py | 92 +++++--- python/setup.py.in | 16 +- tools/parallel_UT_rule.py | 218 +++++++++++++++++- 13 files changed, 405 insertions(+), 175 deletions(-) diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index 24bed27728083..0f85464f60a0f 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -369,36 +369,3 @@ cc_library(paddle_framework DEPS ${FLUID_FRAMEWORK_MODULES}) if(WITH_TESTING AND TEST selected_rows_test) set_tests_properties(selected_rows_test PROPERTIES TIMEOUT 120) endif() - -##### 2.0 New custom op extension mechanism related ##### - -# if not deps `layer`, will cause: undefined symbol: _ZN6paddle10imperative7VarBase9name_set_ -if (WIN32) - set(PADDLE_CUSTOM_OP_MODULES custom_tensor op_meta_info custom_operator layer) - - set(PADDLE_CUSTOM_OP_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/custom_operator.cc - ${CMAKE_CURRENT_SOURCE_DIR}/../extension/src/ext_tensor.cc - ${CMAKE_CURRENT_SOURCE_DIR}/../extension/src/ext_op_meta_info.cc - ${CMAKE_SOURCE_DIR}/paddle/fluid/imperative/layer.cc) - set(PADDLE_CUSTOM_OP_SRCS ${PADDLE_CUSTOM_OP_SRCS} PARENT_SCOPE) - - cc_library(paddle_custom_op_shared - SHARED SRCS ${PADDLE_CUSTOM_OP_SRCS} DEPS ${PADDLE_CUSTOM_OP_MODULES}) - - get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES) - set_target_properties(paddle_custom_op_shared PROPERTIES OUTPUT_NAME paddle_custom_op) - target_link_libraries(paddle_custom_op_shared ${os_dependency_modules}) - - if("${CMAKE_GENERATOR}" STREQUAL "Ninja") - set(paddle_custom_op_lib_path ${CMAKE_CURRENT_BINARY_DIR}) - else() - set(paddle_custom_op_lib_path ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}) - endif() - set(PADDLE_CUSTOM_OP_IMPORT_LIB - ${paddle_custom_op_lib_path}/paddle_custom_op.lib - CACHE INTERNAL "Paddle custom op import lib") - set(PADDLE_CUSTOM_OP_SHARED_LIB - ${paddle_custom_op_lib_path}/paddle_custom_op.dll - CACHE INTERNAL "Paddle custom op dll") -endif() diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 439c8a4f24189..e53828ff10be6 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -54,14 +54,14 @@ wmic process where name="python.exe" call terminate 2>NUL rem ------initialize common variable------ if not defined GENERATOR set GENERATOR="Visual Studio 15 2017 Win64" if not defined BRANCH set BRANCH=develop -if not defined WITH_TENSORRT set WITH_TENSORRT=ON +if not defined WITH_TENSORRT set WITH_TENSORRT=ON if not defined TENSORRT_ROOT set TENSORRT_ROOT=D:/TensorRT if not defined CUDA_ARCH_NAME set CUDA_ARCH_NAME=Auto if not defined WITH_GPU set WITH_GPU=ON if not defined WITH_MKL set WITH_MKL=ON if not defined WITH_AVX set WITH_AVX=ON if not defined WITH_TESTING set WITH_TESTING=ON -if not defined MSVC_STATIC_CRT set MSVC_STATIC_CRT=OFF +if not defined MSVC_STATIC_CRT set MSVC_STATIC_CRT=ON if not defined WITH_PYTHON set WITH_PYTHON=ON if not defined ON_INFER set ON_INFER=ON if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=ON @@ -75,6 +75,7 @@ if not defined LOG_LEVEL set LOG_LEVEL=normal if not defined PRECISION_TEST set PRECISION_TEST=OFF if not defined NIGHTLY_MODE set PRECISION_TEST=OFF if not defined retry_times set retry_times=2 +if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37 rem -------set cache build directory----------- rmdir build\python /s/q @@ -83,9 +84,6 @@ rmdir build\paddle_inference_install_dir /s/q rmdir build\paddle_inference_c_install_dir /s/q del build\CMakeCache.txt -: set CI_SKIP_CPP_TEST if only *.py changed -git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON - if "%WITH_CACHE%"=="OFF" ( rmdir build /s/q goto :mkbuild @@ -135,58 +133,6 @@ dir . dir %cache_dir% dir paddle\fluid\pybind\Release -rem ------initialize the python environment------ -if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37 -set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe -set PATH=%PYTHON_ROOT%;%PYTHON_ROOT%\Scripts;%PATH% - -rem ToDo: virtual environment can't be deleted safely, some process not exit when task is canceled -rem Now use system python environment temporarily -rem %PYTHON_EXECUTABLE% -m pip install virtualenv -rem %PYTHON_EXECUTABLE% -m virtualenv paddle_winci -rem call paddle_winci\Scripts\activate.bat - -rem ------pre install python requirement---------- -where python -where pip -pip install wheel --user -pip install -r %work_dir%\python\requirements.txt --user - -if %ERRORLEVEL% NEQ 0 ( - echo pip install requirements.txt failed! - exit /b 7 -) - -rem ------pre install clcache and init config---------- -rem pip install clcache --user -pip uninstall -y clcache -:: set USE_CLCACHE to enable clcache -rem set USE_CLCACHE=1 -:: In some scenarios, CLCACHE_HARDLINK can save one file copy. -rem set CLCACHE_HARDLINK=1 -:: If it takes more than 1000s to obtain the right to use the cache, an error will be reported -rem set CLCACHE_OBJECT_CACHE_TIMEOUT_MS=1000000 -:: set maximum cache size to 20G -rem clcache.exe -M 21474836480 - -:: install ninja if GENERATOR is Ninja -if %GENERATOR% == "Ninja" ( - pip install ninja - if %errorlevel% NEQ 0 ( - echo pip install ninja failed! - exit /b 7 - ) -) - -rem ------show summary of current environment---------- -cmake --version -if "%WITH_GPU%"=="ON" ( - nvcc --version - nvidia-smi -) -::python %work_dir%\tools\summary_env.py -::%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\get_cpu_info.sh - goto :CASE_%1 echo "Usage: paddle_build.bat [OPTION]" @@ -266,8 +212,10 @@ rem "Other configurations are added here" rem :CASE_wincheck_others rem call ... + rem --------------------------------------------------------------------------------------------- :cmake +@ECHO OFF echo ======================================== echo Step 1. Cmake ... echo ======================================== @@ -281,12 +229,52 @@ set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH% for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%# set start=%start:~4,10% -@ECHO ON -if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0 +if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2 set PATH=%TENSORRT_ROOT:/=\%\lib;%CUDA_TOOLKIT_ROOT_DIR%\bin;%CUDA_TOOLKIT_ROOT_DIR%\libnvvp;%PATH% -rem ------set third_party cache dir------ +rem install ninja if GENERATOR is Ninja +if %GENERATOR% == "Ninja" ( + pip install ninja + if %errorlevel% NEQ 0 ( + echo pip install ninja failed! + exit /b 7 + ) +) +rem ------show summary of current GPU environment---------- +cmake --version +if "%WITH_GPU%"=="ON" ( + nvcc --version + nvidia-smi +) + +rem ------initialize the python environment------ +set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe +set PATH=%PYTHON_ROOT%;%PYTHON_ROOT%\Scripts;%PATH% +if %WITH_PYTHON% == "OFF" ( + where python + where pip + pip install wheel --user + pip install -r %work_dir%\python\requirements.txt --user + if %ERRORLEVEL% NEQ 0 ( + echo pip install requirements.txt failed! + exit /b 7 + ) +) + +rem ------pre install clcache and init config---------- +rem pip install clcache --user +pip uninstall -y clcache +:: set USE_CLCACHE to enable clcache +rem set USE_CLCACHE=1 +:: In some scenarios, CLCACHE_HARDLINK can save one file copy. +rem set CLCACHE_HARDLINK=1 +:: If it takes more than 1000s to obtain the right to use the cache, an error will be reported +rem set CLCACHE_OBJECT_CACHE_TIMEOUT_MS=1000000 +:: set maximum cache size to 20G +rem clcache.exe -M 21474836480 + +rem ------set third_party cache dir------ : clear third party cache every once in a while for /F %%# in ('wmic os get localdatetime^|findstr 20') do set datetime=%%# set day_now=%datetime:~6,2% @@ -500,6 +488,10 @@ echo ======================================== echo Step 4. Running unit tests ... echo ======================================== + +: set CI_SKIP_CPP_TEST if only *.py changed +git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON + pip install -r %work_dir%\python\unittest_py\requirements.txt --user if %ERRORLEVEL% NEQ 0 ( echo pip install unittest requirements.txt failed! diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 9b03cd08ba97a..b493ecedd9651 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -43,9 +43,20 @@ set(FLUID_DST_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/) IF(WIN32) # Python would use the .pyd by default under Windows series platform set(FLUID_CORE ${FLUID_DST_DIR}/${FLUID_CORE_NAME}.pyd) - set(FLUID_NOAVX_CORE ${FLUID_DST_DIR}/core_noavx.pyd) + set(FLUID_CORE_LIB ${FLUID_DST_DIR}/${FLUID_CORE_NAME}.lib) + + add_custom_command(OUTPUT ${FLUID_CORE} + COMMAND cmake -E copy $ ${FLUID_CORE} + COMMAND cmake -E copy $ ${FLUID_CORE_LIB} + DEPENDS paddle_pybind) + + set(FLUID_NOAVX_CORE ${FLUID_DST_DIR}/core_noavx.pyd) ELSE() set(FLUID_CORE ${FLUID_DST_DIR}/${FLUID_CORE_NAME}.so) + add_custom_command(OUTPUT ${FLUID_CORE} + COMMAND cmake -E copy $ ${FLUID_CORE} + DEPENDS paddle_pybind) + set(FLUID_NOAVX_CORE ${FLUID_DST_DIR}/core_noavx.so) ENDIF() @@ -68,9 +79,6 @@ if(HAS_NOAVX_CORE AND EXISTS "${NOAVX_CORE_FILE}") list(APPEND FLUID_CORE_DEPS ${FLUID_NOAVX_CORE}) endif() -add_custom_command(OUTPUT ${FLUID_CORE} - COMMAND cmake -E copy $ ${FLUID_CORE} - DEPENDS paddle_pybind) add_custom_target(copy_paddle_pybind ALL DEPENDS ${FLUID_CORE_DEPS}) IF(WIN32) @@ -84,6 +92,7 @@ ELSE(WIN32) COMMAND touch stub.cc COMMAND cp -r ${PADDLE_SOURCE_DIR}/python/paddle ${PADDLE_BINARY_DIR}/python COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel + COMMENT "Packing whl packages------>>>" DEPENDS copy_paddle_pybind ${FLUID_CORE} framework_py_proto profiler_py_proto ${PY_FILES}) ENDIF() diff --git a/python/paddle/check_import_scipy.py b/python/paddle/check_import_scipy.py index 0172d568e5b08..d6e13e2a67085 100644 --- a/python/paddle/check_import_scipy.py +++ b/python/paddle/check_import_scipy.py @@ -24,6 +24,6 @@ def check_import_scipy(OsName): if 'DLL load failed' in print_info: raise ImportError( print_info + - "\nplease download visual C++ Redistributable for vs 2015, https://www.microsoft.com/en-us/download/details.aspx?id=48145" + "\nplease download Visual C++ Redistributable from https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0" ) return diff --git a/python/paddle/fluid/core.py b/python/paddle/fluid/core.py index 49bcaf6dd608c..9e931ad40c57a 100644 --- a/python/paddle/fluid/core.py +++ b/python/paddle/fluid/core.py @@ -37,7 +37,10 @@ try: if os.name == 'nt': third_lib_path = current_path + os.sep + '..' + os.sep + 'libs' - os.environ['path'] = third_lib_path + ';' + os.environ['path'] + # Will load shared library from 'path' on windows + os.environ[ + 'path'] = current_path + ';' + third_lib_path + ';' + os.environ[ + 'path'] sys.path.insert(0, third_lib_path) # Note: from python3.8, PATH will not take effect # https://github.com/python/cpython/pull/12302 @@ -298,7 +301,7 @@ def to_list(s): "WARNING: AVX is supported on local machine, but you have installed " "paddlepaddle without avx core. Hence, no_avx core which has worse " "preformance will be imported.\nYou could reinstall paddlepaddle by " - "'python -m pip install -U paddlepaddle-gpu[==version]' or rebuild " + "'python -m pip install --force-reinstall paddlepaddle-gpu[==version]' or rebuild " "paddlepaddle WITH_AVX=ON to get better performance.\n" "The original error is: %s\n" % cpt.get_exception_message(e)) load_noavx = True @@ -350,12 +353,19 @@ def to_list(s): sys.stderr.write( 'Error: Can not import noavx core while this file exists: ' + current_path + os.sep + 'core_noavx.' + core_suffix + '\n') + elif avx_supported(): + sys.stderr.write( + "Error: AVX is support on your machine, but you have installed " + "paddlepaddle without avx core, you should reinstall paddlepaddle by " + "'python -m pip install --force-reinstall paddlepaddle-gpu[==version]\n" + ) else: sys.stderr.write( "Error: AVX is not support on your machine, but you have installed " - "paddlepaddle with avx core, you should reinstall paddlepaddle by " - "'python -m pip install -U paddlepaddle-gpu[==version] -f " - "https://paddlepaddle.org.cn/whl/stable_noavx.html'\n") + "paddlepaddle without no_avx core, you should reinstall paddlepaddle by " + "'python -m pip install --force-reinstall paddlepaddle-gpu[==version] -f " + "https://paddlepaddle.org.cn/whl/mkl/stable/noavx.html or " + "https://paddlepaddle.org.cn/whl/openblas/stable/noavx.html\n") raise e diff --git a/python/paddle/fluid/tests/custom_op/CMakeLists.txt b/python/paddle/fluid/tests/custom_op/CMakeLists.txt index 81f64038c7c90..2092151b84f45 100644 --- a/python/paddle/fluid/tests/custom_op/CMakeLists.txt +++ b/python/paddle/fluid/tests/custom_op/CMakeLists.txt @@ -1,6 +1,5 @@ -# New custom OP can support Windows/Linux now -if(WITH_GPU OR APPLE) - # GPU custom op tests: compile both .cc and .cu file +# New custom OP can support Windows/Linux/Mac now +if(WITH_GPU OR APPLE) py_test(test_custom_relu_op_setup SRCS test_custom_relu_op_setup.py) py_test(test_custom_relu_op_jit SRCS test_custom_relu_op_jit.py) py_test(test_custom_relu_model SRCS test_custom_relu_model.py) diff --git a/python/paddle/fluid/tests/custom_op/custom_relu_op.cu b/python/paddle/fluid/tests/custom_op/custom_relu_op.cu index 4ec7d0884582e..38e8e71cf8129 100644 --- a/python/paddle/fluid/tests/custom_op/custom_relu_op.cu +++ b/python/paddle/fluid/tests/custom_op/custom_relu_op.cu @@ -45,8 +45,12 @@ std::vector relu_cuda_forward(const paddle::Tensor& x) { int grid = (numel + block - 1) / block; PD_DISPATCH_FLOATING_AND_HALF_TYPES( x.type(), "relu_cuda_forward_kernel", ([&] { + auto cpu_input = x.copy_to(paddle::PlaceType::kCPU); + auto gpu_input = cpu_input.copy_to(paddle::PlaceType::kGPU); relu_cuda_forward_kernel<<>>( - x.data(), out.mutable_data(x.place()), numel); + gpu_input.data(), + out.mutable_data(x.place()), + numel); })); return {out}; diff --git a/python/paddle/fluid/tests/custom_op/test_check_abi.py b/python/paddle/fluid/tests/custom_op/test_check_abi.py index 75cf99458e71a..baef25d2d1162 100644 --- a/python/paddle/fluid/tests/custom_op/test_check_abi.py +++ b/python/paddle/fluid/tests/custom_op/test_check_abi.py @@ -64,14 +64,29 @@ def test_wrong_compiler_warning(self): # clear environ self.del_environ() compiler = 'python' # fake wrong compiler - with warnings.catch_warnings(record=True) as error: - flag = utils.check_abi_compatibility(compiler, verbose=True) - # check return False - self.assertFalse(flag) - # check Compiler Compatibility WARNING - self.assertTrue(len(error) == 1) - self.assertTrue( - "Compiler Compatibility WARNING" in str(error[0].message)) + if not utils.IS_WINDOWS: + with warnings.catch_warnings(record=True) as error: + flag = utils.check_abi_compatibility(compiler, verbose=True) + # check return False + self.assertFalse(flag) + # check Compiler Compatibility WARNING + self.assertTrue(len(error) == 1) + self.assertTrue( + "Compiler Compatibility WARNING" in str(error[0].message)) + + def test_exception_windows(self): + # clear environ + self.del_environ() + compiler = 'fake compiler' # fake command + if utils.IS_WINDOWS: + with warnings.catch_warnings(record=True) as error: + flag = utils.check_abi_compatibility(compiler, verbose=True) + # check return False + self.assertFalse(flag) + # check ABI Compatibility WARNING + self.assertTrue(len(error) == 1) + self.assertTrue("Failed to check compiler version for" in + str(error[0].message)) def test_exception_linux(self): # clear environ diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py index d8dcc76ac6067..0f7ba84ffc147 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py @@ -105,12 +105,12 @@ def test_exception(self): in str(e)) if IS_WINDOWS: self.assertTrue( - r"python\paddle\fluid\tests\custom_op\custom_relu_op.cc:47" - in str(e)) + r"python\paddle\fluid\tests\custom_op\custom_relu_op.cc" in + str(e)) else: self.assertTrue( - "python/paddle/fluid/tests/custom_op/custom_relu_op.cc:47" - in str(e)) + "python/paddle/fluid/tests/custom_op/custom_relu_op.cc" in + str(e)) self.assertTrue(caught_exception) caught_exception = False @@ -126,7 +126,7 @@ def test_exception(self): "function \"relu_cuda_forward_kernel\" is not implemented for data type `int32_t`" in str(e)) self.assertTrue( - "python/paddle/fluid/tests/custom_op/custom_relu_op.cu:50" in + "python/paddle/fluid/tests/custom_op/custom_relu_op.cu" in str(e)) self.assertTrue(caught_exception) diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index ab528cdb0c0d9..6045ac7d1e727 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -26,7 +26,7 @@ from .extension_utils import is_cuda_file, prepare_unix_cudaflags, prepare_win_cudaflags from .extension_utils import _import_module_from_library, _write_setup_file, _jit_compile from .extension_utils import check_abi_compatibility, log_v, CustomOpInfo, parse_op_name_from -from .extension_utils import clean_object_if_change_cflags, _reset_so_rpath +from .extension_utils import clean_object_if_change_cflags, _reset_so_rpath, _get_fluid_path from .extension_utils import bootstrap_context, get_build_directory, add_std_without_repeat from .extension_utils import IS_WINDOWS, OS_NAME, MSVC_COMPILE_FLAGS, MSVC_COMPILE_FLAGS @@ -69,7 +69,7 @@ def setup(**attr): For Linux, GCC version will be checked . For example if Paddle with CUDA 10.1 is built with GCC 8.2, then the version of user's local machine should satisfy GCC >= 8.2. For Windows, Visual Studio version will be checked, and it should be greater than or equal to that of - PaddlePaddle (Visual Studio 2015 update3). + PaddlePaddle (Visual Studio 2017). If the above conditions are not met, the corresponding warning will be printed, and a fatal error may occur because of ABI compatibility. @@ -79,7 +79,7 @@ def setup(**attr): 2. On Linux platform, we recommend to use GCC 8.2 as soft linking condidate of ``/usr/bin/cc`` . Then, Use ``which cc`` to ensure location of ``cc`` and using ``cc --version`` to ensure linking GCC version. - 3. On Windows platform, we recommend to install `` Visual Studio`` (>=2015 update3). + 3. On Windows platform, we recommend to install `` Visual Studio`` (>=2017). Compared with Just-In-Time ``load`` interface, it only compiles once by executing @@ -611,7 +611,7 @@ def _check_abi(self): msg = ( 'It seems that the VC environment is activated but DISTUTILS_USE_SDK is not set.' 'This may lead to multiple activations of the VC env.' - 'Please set `DISTUTILS_USE_SDK=1` and try again.') + 'Please run `set DISTUTILS_USE_SDK=1` and try again.') raise UserWarning(msg) def _record_op_info(self): @@ -724,7 +724,7 @@ def load(name, processes under a individual subprocess. It does not require CMake or Ninja environment. On Linux platform, it requires GCC compiler whose version is greater than 5.4 and it should be soft linked to ``/usr/bin/cc`` . On Windows - platform, it requires Visual Studio whose version is greater than 2015 update3. + platform, it requires Visual Studio whose version is greater than 2017. On MacOS, clang++ is requited. In addition, if compiling Operators supporting GPU device, please make sure ``nvcc`` compiler is installed in local environment. @@ -735,7 +735,7 @@ def load(name, For Linux, GCC version will be checked . For example if Paddle with CUDA 10.1 is built with GCC 8.2, then the version of user's local machine should satisfy GCC >= 8.2. For Windows, Visual Studio version will be checked, and it should be greater than or equal to that of - PaddlePaddle (Visual Studio 2015 update3). + PaddlePaddle (Visual Studio 2017). If the above conditions are not met, the corresponding warning will be printed, and a fatal error may occur because of ABI compatibility. @@ -749,7 +749,7 @@ def load(name, 2. On Linux platform, we recommend to use GCC 8.2 as soft linking condidate of ``/usr/bin/cc`` . Then, Use ``which cc`` to ensure location of ``cc`` and using ``cc --version`` to ensure linking GCC version. - 3. On Windows platform, we recommend to install `` Visual Studio`` (>=2015 update3). + 3. On Windows platform, we recommend to install `` Visual Studio`` (>=2017). **A simple example:** @@ -802,9 +802,6 @@ def load(name, # ensure to use abs path build_directory = os.path.abspath(build_directory) - # Will load shared library from 'path' on windows - if IS_WINDOWS: - os.environ['path'] = build_directory + ';' + os.environ['path'] log_v("build_directory: {}".format(build_directory), verbose) @@ -827,6 +824,7 @@ def load(name, # write setup.py file and compile it build_base_dir = os.path.join(build_directory, name) + _write_setup_file(name, sources, file_path, build_base_dir, extra_include_paths, extra_cxx_cflags, extra_cuda_cflags, extra_ldflags, verbose) diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index c055084886c25..ea46ea8b39195 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -55,7 +55,7 @@ '-dynamiclib', '-undefined', 'dynamic_lookup', '-arch', 'x86_64' ] -MSVC_LINK_FLAGS = ['/MACHINE:X64', 'paddle_custom_op.lib'] +MSVC_LINK_FLAGS = ['/MACHINE:X64'] COMMON_NVCC_FLAGS = ['-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU'] @@ -371,10 +371,11 @@ def _get_core_name(): Return pybind DSO module name. """ import paddle - if paddle.fluid.core.load_noavx: - return 'core_noavx.so' + ext_name = '.pyd' if IS_WINDOWS else '.so' + if not paddle.fluid.core.load_noavx: + return 'core_avx' + ext_name else: - return 'core_avx.so' + return 'core_noavx' + ext_name def _get_lib_core_path(): @@ -386,6 +387,15 @@ def _get_lib_core_path(): return os.path.join(_get_fluid_path(), lib_core_name) +def _get_dll_core_path(): + """ + Return real path of libcore_(no)avx.dylib on Windows. + """ + raw_core_name = _get_core_name() + dll_core_name = "paddle_pybind.dll" + return os.path.join(_get_fluid_path(), dll_core_name) + + def _reset_so_rpath(so_path): """ NOTE(Aurelius84): Runtime path of core_(no)avx.so is modified into `@loader_path/../libs` @@ -435,9 +445,12 @@ def normalize_extension_kwargs(kwargs, use_cuda=False): # append link flags extra_link_args = kwargs.get('extra_link_args', []) extra_link_args.extend(MSVC_LINK_FLAGS) + lib_core_name = create_sym_link_if_not_exist() + extra_link_args.append('{}'.format(lib_core_name)) if use_cuda: extra_link_args.extend(['cudadevrt.lib', 'cudart_static.lib']) kwargs['extra_link_args'] = extra_link_args + else: ########################### Linux Platform ########################### extra_link_args = kwargs.get('extra_link_args', []) @@ -481,24 +494,41 @@ def create_sym_link_if_not_exist(): """ Create soft symbol link of `core_avx.so` or `core_noavx.so` """ - assert OS_NAME.startswith('darwin') + assert OS_NAME.startswith('darwin') or IS_WINDOWS raw_core_name = _get_core_name() core_path = os.path.join(_get_fluid_path(), raw_core_name) - new_lib_core_path = _get_lib_core_path() + if IS_WINDOWS: + new_dll_core_path = _get_dll_core_path() + # create symbol link on windows + if not os.path.exists(new_dll_core_path): + try: + os.symlink(core_path, new_dll_core_path) + except Exception: + warnings.warn( + "Failed to create soft symbol link for {}.\n You can run prompt as administrator and execute the " + "following command manually: `mklink {} {}`. Now it will create hard link for {} trickly.". + format(raw_core_name, new_dll_core_path, core_path, + raw_core_name)) + run_cmd('mklink /H {} {}'.format(new_dll_core_path, core_path)) + # core_avx or core_noavx with lib suffix + assert os.path.exists(new_dll_core_path) + return raw_core_name[:-4] + ".lib" - # create symbol link - if not os.path.exists(new_lib_core_path): - try: - os.symlink(core_path, new_lib_core_path) - assert os.path.exists(new_lib_core_path) - except Exception: - raise RuntimeError( - "Failed to create soft symbol link for {}.\n Please execute the following command manually: `ln -s {} {}`". - format(raw_core_name, core_path, new_lib_core_path)) + else: + new_lib_core_path = _get_lib_core_path() + # create symbol link on mac + if not os.path.exists(new_lib_core_path): + try: + os.symlink(core_path, new_lib_core_path) + assert os.path.exists(new_lib_core_path) + except Exception: + raise RuntimeError( + "Failed to create soft symbol link for {}.\n Please execute the following command manually: `ln -s {} {}`". + format(raw_core_name, core_path, new_lib_core_path)) - # core_avx or core_noavx without suffix - return raw_core_name[:-3] + # core_avx or core_noavx without suffix + return raw_core_name[:-3] def find_cuda_home(): @@ -1054,20 +1084,20 @@ def check_abi_compatibility(compiler, verbose=False): if os.environ.get('PADDLE_SKIP_CHECK_ABI') in ['True', 'true', '1']: return True - which = 'where' if IS_WINDOWS else 'which' - cmd_out = subprocess.check_output( - [which, compiler], stderr=subprocess.STDOUT) - compiler_path = os.path.realpath(cmd_out.decode() - if six.PY3 else cmd_out).strip() - # step 1. if not found any suitable compiler, raise error - if not any(name in compiler_path - for name in _expected_compiler_current_platform()): - warnings.warn( - WRONG_COMPILER_WARNING.format( - user_compiler=compiler, - paddle_compiler=_expected_compiler_current_platform()[0], - platform=OS_NAME)) - return False + if not IS_WINDOWS: + cmd_out = subprocess.check_output( + ['which', compiler], stderr=subprocess.STDOUT) + compiler_path = os.path.realpath(cmd_out.decode() + if six.PY3 else cmd_out).strip() + # if not found any suitable compiler, raise warning + if not any(name in compiler_path + for name in _expected_compiler_current_platform()): + warnings.warn( + WRONG_COMPILER_WARNING.format( + user_compiler=compiler, + paddle_compiler=_expected_compiler_current_platform()[0], + platform=OS_NAME)) + return False version = (0, 0, 0) # clang++ have no ABI compatibility problem diff --git a/python/setup.py.in b/python/setup.py.in index d9ca3038fb2b7..0f2e97192c1df 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -255,11 +255,15 @@ paddle_bins = '' if not '${WIN32}': paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle'] -package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + ('.so' if os.name != 'nt' else '.pyd')]} + +if os.name != 'nt': + package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.so']} +else: + package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.pyd', '${FLUID_CORE_NAME}' + '.lib']} + if '${HAS_NOAVX_CORE}' == 'ON': package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')] - package_dir={ '': '${PADDLE_BINARY_DIR}/python', # The paddle.fluid.proto will be generated while compiling. @@ -353,14 +357,6 @@ if '${WITH_XPU}' == 'OFF' and '${XPU_SDK_ROOT}' != '': package_data['paddle.libs']+=['libxpurt.so'] -### New custom op extension mechanism related ### - -# copy paddle_custom_op.lib/paddle_custom_op.dll to libs on Windows -if os.name == 'nt': - shutil.copy('${PADDLE_CUSTOM_OP_IMPORT_LIB}', libs_path) - shutil.copy('${PADDLE_CUSTOM_OP_SHARED_LIB}', libs_path) - package_data['paddle.libs'] += ['paddle_custom_op.lib', 'paddle_custom_op.dll'] - # remove unused paddle/libs/__init__.py if os.path.isfile(libs_path+'/__init__.py'): os.remove(libs_path+'/__init__.py') diff --git a/tools/parallel_UT_rule.py b/tools/parallel_UT_rule.py index d2969618b85e8..9d03ae22de28f 100644 --- a/tools/parallel_UT_rule.py +++ b/tools/parallel_UT_rule.py @@ -436,9 +436,172 @@ 'assign_op_test', 'allocator_facade_frac_flags_test', 'aes_cipher_test', + 'test_dist_sparse_tensor_load_adagrad', + 'test_dist_mnist_fp16_allreduce', + 'test_dist_mnist_gradient_merge', + 'test_dist_allreduce_op', + 'test_hdfs3', + 'test_parallel_dygraph_se_resnext', + 'test_dist_fleet_ps9', + 'test_dist_fleet_infer', + 'test_dist_se_resnext_sync', + 'test_dist_oneps', + 'test_dist_sparse_load_ps1', + 'test_dist_mnist_batch_merge', + 'test_dist_fleet_ctr', + 'test_dist_fleet_ps10', + 'test_parallel_dygraph_transformer', + 'test_dist_mnist_fleetapi', + 'test_dist_sparse_tensor_load_adam', + 'test_dist_fleet_ps4', + 'test_dist_fleet_heter_program', + 'test_parallel_dygraph_sparse_embedding_over_height', + 'test_hdfs2', + 'test_dist_sharding_save', + 'test_dist_fleet_ps_gpu_ctr', + 'test_dist_mnist_backward_deps', + 'test_dist_fleet_heter_base', + 'test_dist_sparse_tensor_load_sgd', + 'test_new_group', + 'test_dist_mnist_with_program', + 'test_dist_mnist_pg', + 'test_dist_sparse_tensor_load_rmsprop', + 'test_auto_checkpoint2', + 'test_dist_sparse_tensor_load_ftrl', + 'test_dist_fleet_ps6', + 'test_dist_mnist_fleet_save', + 'test_auto_checkpoint1', + 'test_dist_fleet_a_sync_optimizer_sync', + 'test_dist_fleet_ps3', + 'test_dist_se_resnext_nccl', + 'test_parallel_dygraph_mnist', + 'test_auto_checkpoint_multiple', + 'test_dist_fleet_a_sync_optimizer_auto_async', + 'test_pipeline', + 'test_dist_fleet_ps8', + 'test_dist_fleet_sparse_embedding_ctr', + 'test_dist_se_resnext_dgc', + 'test_dist_fleet_ps7', + 'test_dist_fleet_decay', + 'test_dist_fleet_a_sync_optimizer_auto_geo', + 'test_dist_fleet_geo', + 'test_parallel_dygraph_dataparallel', + 'test_hdfs1', + 'test_dist_mnist_dgc_nccl', + 'test_dist_fleet_ctr2', + 'test_parallel_dygraph_unused_variables', + 'test_dist_mnist_multi_comm', + 'test_dist_sparse_tensor_load_momentum', + 'test_gen_nccl_id_op', + 'test_parallel_dygraph_sparse_embedding', + 'test_dist_mnist_ring_allreduce', + 'test_fleet_launch_async', + 'test_dist_fleet_a_sync_optimizer_geo', + 'test_parallel_dygraph_control_flow', + 'test_auto_checkpoint', + 'test_fleet_pipeline_meta_optimizer', + 'test_dist_fleet_heter_ctr', + 'test_fleet_graph_execution_meta_optimizer', + 'test_fleet_run_random_port', + 'test_dist_fleet_ps5', + 'test_dist_fleet_a_sync_optimizer_auto', + 'test_dist_lookup_sparse_table_fuse_ops', + 'test_dist_fleet_a_sync_optimizer_async', + 'test_c_comm_init_op', + 'test_fleet_launch_nproc', + 'test_dist_fleet_simnet', + 'test_auto_checkpoint_dist_basic', + 'test_fleet_launch_cloud', + 'test_dist_fleet_ps', + 'test_dist_op', + 'test_dist_sparse_load_ps0', + 'test_auto_checkpoint3', + 'test_dist_fleet_ps2', + 'test_dist_fleet_grad_clip', + 'test_custom_concat', + 'test_analyzer_transformer_fuse', + 'test_analyzer_seq_pool1_fuse_statis', + 'test_fc_lstm_fuse_pass_cc', + 'test_layer_norm_fuse_pass', + 'test_fc_gru_fuse_pass_cc', + 'test_analyzer_save_model', + 'test_fleet_ps', + 'test_analyzer_multi_model_prediction', + 'test_fleet_base_3', + 'test_fleet_base_2', + 'test_ascend_trigger', + 'test_fleet_amp_meta_optimizer', + 'test_fleetrun', + 'test_check_abi', + 'dense_table_test', + 'test_adaptive_pool2d_convert_global_pass', + 'test_fleet_recompute_meta_optimizer', + 'test_fleet_fp16_allreduce_meta_optimizer', + 'test_post_training_quantization_lstm_model', + 'test_fleet_metric', + 'test_fleet_gradient_merge_meta_optimizer', + 'test_fleet_sharding_meta_optimizer', + 'test_listen_and_serv_op', + 'test_analyzer_zerocopytensor_tensor', + 'test_conv_bn_fuse_pass_cc', + 'test_collective_optimizer', + 'test_bf16_utils', + 'test_analyzer_seq_pool1_compare_determine', + 'test_avoid_twice_initialization', + 'test_callback_early_stop', + 'test_fleet_distributed_strategy', + 'test_launch_coverage', + 'test_sgd_op_bf16', + 'test_model_cast_to_bf16', + 'test_hybrid_parallel_topology', + 'barrier_table_test', + 'test_check_error', + 'test_fleet_lamb_meta_optimizer', + 'test_fleet_rolemaker_2', + 'test_distributed_strategy', + 'test_rnn_cudnn_params_packing', + 'test_communicator_async', + 'brpc_utils_test', + 'test_analyzer_capi_pd_tensor', + 'test_recv_save_op', + 'heter_listen_and_server_test', + 'test_analyzer_capi_ner', + 'test_unsqueeze2_eltwise_fuse_pass', + 'test_dgc_optimizer', + 'test_fleet_cc', + 'test_repeated_fc_relu_fuse_pass_cc', + 'heter_server_test', + 'test_static_save_load_large', + 'graph_node_test', + 'test_custom_conj', + 'test_fleet_private_function', + 'test_fake_init_op', + 'brpc_service_sparse_sgd_test', + 'test_tf32_cudnn', + 'test_communicator_geo', + 'test_dispatch_jit', + 'test_layer_norm_fuse_pass_cc', + 'test_fleet_dgc_meta_optimizer', + 'test_fc_fuse_pass_cc', + 'test_communicator_sync', + 'test_analyzer_capi', + 'test_fleet_lars_meta_optimizer', + 'test_communicator_half_async', + 'test_fleet_localsgd_meta_optimizer', + 'test_fleet_amp_init', + 'test_fleet_checkpoint', + 'test_analyzer_seq_pool1_fuse_compare_zero_copy', + 'test_lookup_table_bf16_op', + 'test_fleet_meta_optimizer_base', + 'table_test', + 'test_fleet_rolemaker_new', + 'test_fleet_graph_executor', + 'test_multi_out_jit', + 'test_fleet_utils', + 'brpc_service_dense_sgd_test', ] -# It run 4 job each time, If it failed due to Insufficient GPU memory or CUBLAS_STATUS_ALLOC_FAILED, +# It run 4 job each time, If it failed due to Insufficient GPU memory or CUBLAS_STATUS_ALLOC_FAILED, # just remove it from this list. TETRAD_PARALLEL_JOB = [ 'buffered_allocator_test', @@ -477,9 +640,53 @@ 'tensor_test', 'test_repeated_fc_relu_fuse_pass_cc', 'test_mkldnn_caching', + 'test_analyzer_seq_pool1', + 'test_analyzer_ocr', + 'test_analyzer_seq_conv1', + 'test_analyzer_small_dam', + 'test_analyzer_mobilenet_depthwise_conv', + 'test_analyzer_pyramid_dnn', + 'test_analyzer_text_classification', + 'test_analyzer_rnn2', + 'test_analyzer_transformer', + 'test_analyzer_resnet50', + 'test_analyzer_ner', + 'test_analyzer_lac', + 'test_analyzer_transformer_profile', + 'test_analyzer_mobilenet_transpose', + 'test_analyzer_rnn1', + 'test_analyzer_seq_pool1_profile', + 'test_analyzer_paddletensor_tensor', + 'test_analyzer_bert', + 'test_analyzer_googlenet', + 'zero_copy_tensor_test', + 'custom_tensor_test', + 'test_fleet_base', + 'test_imperative_container_layerdict', + 'test_complex_simplenet', + 'test_tensor_register_hook', + 'test_set_value_op', + 'test_tensor_type_promotion', + 'test_view_op_reuse_allocation', + 'test_complex_grad_accumulated', + 'test_sequential', + 'test_sequential', + 'test_imperative_layers', + 'test_dgc_momentum_op', + 'test_memcpy_op', + 'test_dgc_op', + 'test_modelaverage', + 'test_lookahead', + 'test_callback_visualdl', + 'test_new_group_api', + 'test_collective_split_embedding_none_divisible', + 'test_collective_wait', + 'test_collective_split_row_linear', + 'test_collective_split_col_linear', + 'test_collective_split_embedding', ] -# It run 2 job each time, If it failed due to Insufficient GPU memory or CUBLAS_STATUS_ALLOC_FAILED, +# It run 2 job each time, If it failed due to Insufficient GPU memory or CUBLAS_STATUS_ALLOC_FAILED, # just remove it from this list. TWO_PARALLEL_JOB = [ 'convert_model2dot_ernie', @@ -611,7 +818,6 @@ 'test_adam_op_multi_thread', 'test_adamax_op', 'test_while_loop_op', - 'test_affine_grid_function', 'test_transpose_flatten_concat_fuse_pass', 'test_trace_op', 'test_backward', @@ -663,7 +869,6 @@ 'test_gather_op', 'test_partial_concat_op', 'test_gaussian_random_op', - 'test_paddle_imperative_double_grad', 'test_generate_proposals_v2_op', 'test_pad_constant_like', 'test_grid_sample_function', @@ -879,6 +1084,11 @@ 'test_imperative_load_static_param', 'test_fuse_bn_add_act_pass', 'test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass', + 'test_quantize_transpiler_v2', + 'paddle_infer_api_test', + 'test_analyzer_ernie', + 'lite_resnet50_test', + 'lite_mul_model_test', ] From ded39f84217978d013c192015cdea87968f0af3f Mon Sep 17 00:00:00 2001 From: LielinJiang <50691816+LielinJiang@users.noreply.github.com> Date: Fri, 7 May 2021 16:42:02 +0800 Subject: [PATCH 047/156] [Cherrypick 2.1] fix compile error on jetson platform (#32760) * fix compile error on jetson platform * remove unused head file * rm decode_jpeg op on jetson platform --- cmake/operators.cmake | 3 +++ paddle/fluid/operators/decode_jpeg_op.cc | 1 - paddle/fluid/operators/decode_jpeg_op.cu | 2 +- paddle/fluid/platform/dynload/CMakeLists.txt | 6 +++++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmake/operators.cmake b/cmake/operators.cmake index 16288e1fb45df..00cf2318f8f78 100644 --- a/cmake/operators.cmake +++ b/cmake/operators.cmake @@ -44,6 +44,9 @@ function(op_library TARGET) if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu) list(APPEND cu_srcs ${TARGET}.cu) endif() + if (WITH_NV_JETSON) + list(REMOVE_ITEM cu_srcs "decode_jpeg_op.cu") + endif() if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu) set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE) diff --git a/paddle/fluid/operators/decode_jpeg_op.cc b/paddle/fluid/operators/decode_jpeg_op.cc index e553b1076a864..dd82c74885b94 100644 --- a/paddle/fluid/operators/decode_jpeg_op.cc +++ b/paddle/fluid/operators/decode_jpeg_op.cc @@ -19,7 +19,6 @@ #include "paddle/fluid/framework/generator.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/operator.h" -#include "paddle/fluid/platform/dynload/nvjpeg.h" #include "paddle/fluid/platform/enforce.h" namespace paddle { diff --git a/paddle/fluid/operators/decode_jpeg_op.cu b/paddle/fluid/operators/decode_jpeg_op.cu index 35975a6a54986..11616b0e0c4da 100644 --- a/paddle/fluid/operators/decode_jpeg_op.cu +++ b/paddle/fluid/operators/decode_jpeg_op.cu @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef PADDLE_WITH_HIP +#if !defined(WITH_NV_JETSON) && !defined(PADDLE_WITH_HIP) #include #include "paddle/fluid/framework/op_registry.h" diff --git a/paddle/fluid/platform/dynload/CMakeLists.txt b/paddle/fluid/platform/dynload/CMakeLists.txt index 8bff2ead0a2a3..21d9e8607459a 100644 --- a/paddle/fluid/platform/dynload/CMakeLists.txt +++ b/paddle/fluid/platform/dynload/CMakeLists.txt @@ -1,6 +1,10 @@ cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags enforce) -list(APPEND CUDA_SRCS cublas.cc cudnn.cc curand.cc cusolver.cc nvtx.cc nvjpeg.cc) +list(APPEND CUDA_SRCS cublas.cc cudnn.cc curand.cc cusolver.cc nvtx.cc) + +if (NOT WITH_NV_JETSON) + list(APPEND CUDA_SRCS nvjpeg.cc) +endif() if (WITH_ROCM) list(APPEND HIP_SRCS rocblas.cc miopen.cc hiprand.cc) From f54fb1eeea759d2657be1aabd87641ec12dce89c Mon Sep 17 00:00:00 2001 From: Jiawei Wang Date: Fri, 7 May 2021 19:51:40 +0800 Subject: [PATCH 048/156] fix stack grad gpu (#32781) --- paddle/fluid/operators/stack_op.cu | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/operators/stack_op.cu b/paddle/fluid/operators/stack_op.cu index 4800f5f9eb533..9e5e45f4d22d9 100644 --- a/paddle/fluid/operators/stack_op.cu +++ b/paddle/fluid/operators/stack_op.cu @@ -96,9 +96,10 @@ class StackGPUKernel : public framework::OpKernel { }; template -__global__ void UnStackCUDAKernel(const T* __restrict__ input, int pre_dim_size, - int split_dim_size, int suf_dim_size, - int num_split, T** output_ptrs) { +__global__ void UnStackHelperCUDAKernel(const T* __restrict__ input, + int pre_dim_size, int split_dim_size, + int suf_dim_size, int num_split, + T** output_ptrs) { assert(blockDim.y == 1); assert(blockDim.z == 1); // In this case they are equal @@ -114,6 +115,9 @@ __global__ void UnStackCUDAKernel(const T* __restrict__ input, int pre_dim_size, IntType k = offset % suf_dim_size; T* output = output_ptrs[j / each_dim_size]; + if (output == nullptr) { + return; + } IntType output_ind = i * each_dim_size * suf_dim_size + (j % each_dim_size) * suf_dim_size + k; *(output + output_ind) = input[offset]; @@ -142,6 +146,9 @@ class StackGradGPUKernel : public framework::OpKernel { std::vector outputs(n); auto out_var_names = ctx.OutputNames(framework::GradVarName("X")); for (size_t j = 0; j < dx.size(); ++j) { + if (dx[j] == nullptr) { + outputs[j] = nullptr; + } if (out_var_names[j] != framework::kEmptyVarName && dx[j]->numel() != 0UL) { T* ptr = dx[j]->mutable_data(ctx.GetPlace()); @@ -170,13 +177,13 @@ class StackGradGPUKernel : public framework::OpKernel { auto config = GetGpuLaunchConfig1D(dev_ctx, dy_pre * split_dim * dy_suf); if (dy->numel() < std::numeric_limits::max()) { - UnStackCUDAKernel< + UnStackHelperCUDAKernel< T, int32_t><<>>( dy_data, dy_pre, split_dim, dy_suf, split_dim, reinterpret_cast(tmp_out_data->ptr())); } else { - UnStackCUDAKernel< + UnStackHelperCUDAKernel< T, int64_t><<>>( dy_data, dy_pre, split_dim, dy_suf, split_dim, From 957cbe6800db170f079e269bd5963bdb139ea384 Mon Sep 17 00:00:00 2001 From: huangjun12 <2399845970@qq.com> Date: Fri, 7 May 2021 19:52:30 +0800 Subject: [PATCH 049/156] fix ce error message, test=release/2.1 (#32758) --- .../unittests/test_cross_entropy_loss.py | 33 +++++++++++++++++++ python/paddle/nn/functional/loss.py | 7 ++++ 2 files changed, 40 insertions(+) diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py b/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py index 897d76a35dcab..a89d47d351d00 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py @@ -20,6 +20,7 @@ import unittest from test_softmax_op import stable_softmax from test_softmax_with_cross_entropy_op import cross_entropy +from paddle.fluid import Program, program_guard def stable_softmax(x): @@ -1363,5 +1364,37 @@ def test_cross_entropy_loss_2d_sum(self): self.assertTrue(np.allclose(dy_ret_value, expected)) +class TestCrossEntropyFAPIError(unittest.TestCase): + def test_errors(self): + with program_guard(Program(), Program()): + + def test_LabelValue(): + input_data = paddle.rand(shape=[20, 100]) + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64") + label_data[0] = 255 + weight_data = paddle.rand([100]) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=255) + + self.assertRaises(ValueError, test_LabelValue) + + def test_LabelValueNeg(): + input_data = paddle.rand(shape=[20, 100]) + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64") + label_data[0] = -1 + weight_data = paddle.rand([100]) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=-1) + + self.assertRaises(ValueError, test_LabelValueNeg) + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index aa0bd8a8c5e3d..eeb0062587646 100755 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -1411,6 +1411,13 @@ def cross_entropy(input, out = core.ops.elementwise_mul(out, weight_gather_reshape) else: + label_min = paddle.min(label) + label_max = paddle.max(label) + if label_min < 0 or label_max >= input.shape[-1]: + raise ValueError( + 'Expected 0 <= label_value < class_dimension({}), but got {} <= label_value <= {} '. + format(input.shape[-1], + label_min.numpy(), label_max.numpy())) weight_gather = core.ops.gather_nd(weight, label) input_shape = list(label.shape) weight_gather_reshape = reshape( From 2ec6b6f10e61f08a52406bfa3f90e0b5e9dc72f0 Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Fri, 7 May 2021 19:53:51 +0800 Subject: [PATCH 050/156] remove packages in __all__ (#32757) * remove packages in __all__ * create new public api level paddle.callbacks;paddle.hub;paddle.utils.unique_name --- python/paddle/__init__.py | 6 ++---- python/paddle/callbacks.py | 31 ++++++++++++++++++++++++++++++ python/paddle/hapi/callbacks.py | 5 +---- python/paddle/hub.py | 21 ++++++++++++++++++++ python/paddle/nn/__init__.py | 2 -- python/paddle/utils/__init__.py | 11 +++-------- python/paddle/utils/download.py | 2 +- python/paddle/utils/unique_name.py | 21 ++++++++++++++++++++ 8 files changed, 80 insertions(+), 19 deletions(-) create mode 100644 python/paddle/callbacks.py create mode 100644 python/paddle/hub.py create mode 100644 python/paddle/utils/unique_name.py diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 054fcdfcbe651..ee4dcaa897940 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -269,10 +269,10 @@ # high-level api from .hapi import Model # noqa: F401 -from .hapi import callbacks # noqa: F401 +from . import callbacks # noqa: F401 from .hapi import summary # noqa: F401 from .hapi import flops # noqa: F401 -from .hapi import hub # noqa: F401 +from . import hub # noqa: F401 import paddle.text # noqa: F401 import paddle.vision # noqa: F401 @@ -335,10 +335,8 @@ 'unsqueeze_', 'argmax', 'Model', - 'callbacks', 'summary', 'flops', - 'hub', 'sort', 'split', 'logical_and', diff --git a/python/paddle/callbacks.py b/python/paddle/callbacks.py new file mode 100644 index 0000000000000..08fab3e0adb5e --- /dev/null +++ b/python/paddle/callbacks.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .hapi.callbacks import Callback # noqa: F401 +from .hapi.callbacks import ProgBarLogger # noqa: F401 +from .hapi.callbacks import ModelCheckpoint # noqa: F401 +from .hapi.callbacks import VisualDL # noqa: F401 +from .hapi.callbacks import LRScheduler # noqa: F401 +from .hapi.callbacks import EarlyStopping # noqa: F401 +from .hapi.callbacks import ReduceLROnPlateau # noqa: F401 + +__all__ = [ #noqa + 'Callback', + 'ProgBarLogger', + 'ModelCheckpoint', + 'VisualDL', + 'LRScheduler', + 'EarlyStopping', + 'ReduceLROnPlateau' +] diff --git a/python/paddle/hapi/callbacks.py b/python/paddle/hapi/callbacks.py index cd4b35ea29a83..61ae8b42d63a9 100644 --- a/python/paddle/hapi/callbacks.py +++ b/python/paddle/hapi/callbacks.py @@ -25,10 +25,7 @@ from .progressbar import ProgressBar -__all__ = [ - 'Callback', 'ProgBarLogger', 'ModelCheckpoint', 'VisualDL', 'LRScheduler', - 'EarlyStopping', 'ReduceLROnPlateau' -] +__all__ = [] def config_callbacks(callbacks=None, diff --git a/python/paddle/hub.py b/python/paddle/hub.py new file mode 100644 index 0000000000000..acdb28cb6f08d --- /dev/null +++ b/python/paddle/hub.py @@ -0,0 +1,21 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .hapi.hub import list # noqa: F401 +from .hapi.hub import help # noqa: F401 +from .hapi.hub import load # noqa: F401 + +__all__ = [ #noqa + 'list', 'help', 'load' +] diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 4e4669892b0f0..b5a6a5ca07384 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -232,10 +232,8 @@ def weight_norm(*args): 'MaxPool3D', 'AdaptiveMaxPool2D', 'Hardshrink', - 'clip', 'Softplus', 'KLDivLoss', - 'clip_by_norm', 'AvgPool2D', 'L1Loss', 'LeakyReLU', diff --git a/python/paddle/utils/__init__.py b/python/paddle/utils/__init__.py index 40c9d415e11f1..c23841ea8b802 100644 --- a/python/paddle/utils/__init__.py +++ b/python/paddle/utils/__init__.py @@ -19,18 +19,13 @@ from .lazy_import import try_import # noqa: F401 from .op_version import OpLastCheckpointChecker # noqa: F401 from .install_check import run_check # noqa: F401 -from ..fluid.framework import unique_name # noqa: F401 +from . import unique_name # noqa: F401 from ..fluid.framework import require_version # noqa: F401 from . import download # noqa: F401 from . import image_util # noqa: F401 from . import cpp_extension # noqa: F401 -__all__ = [ #noqa - 'deprecated', - 'download', - 'run_check', - 'unique_name', - 'require_version', - 'try_import' +__all__ = [ #noqa + 'deprecated', 'run_check', 'require_version', 'try_import' ] diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index ddd1dad9dbdf5..dda8abeff21c0 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -55,7 +55,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): import logging logger = logging.getLogger(__name__) -__all__ = [] +__all__ = ['get_weights_path_from_url'] WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/hapi/weights") diff --git a/python/paddle/utils/unique_name.py b/python/paddle/utils/unique_name.py new file mode 100644 index 0000000000000..d0d487c933d76 --- /dev/null +++ b/python/paddle/utils/unique_name.py @@ -0,0 +1,21 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..fluid.unique_name import generate # noqa: F401 +from ..fluid.unique_name import switch # noqa: F401 +from ..fluid.unique_name import guard # noqa: F401 + +__all__ = [ #noqa + 'generate', 'switch', 'guard' +] From 09b18a49523aabff81c5d6c0946c237f35162640 Mon Sep 17 00:00:00 2001 From: Shang Zhizhou Date: Fri, 7 May 2021 20:41:43 +0800 Subject: [PATCH 051/156] [Paddle-TRT] Implement MHA fp16 order same as training (#32629) (#32785) * implement MHA order same as training * fix fp16 compile issue on old architecture Co-authored-by: zlsh80826 --- .../tensorrt/plugin/qkv_to_context_plugin.cu | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu index a5fc9e73c5f27..214e1a81e7dc0 100644 --- a/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu @@ -225,6 +225,14 @@ nvinfer1::DataType QkvToContextPluginDynamic::getOutputDataType( return input_types[0]; } +template +__global__ void apply_scale(T *data, T scale, int n) { +#if CUDA_ARCH_FP16_SUPPORTED(__CUDA_ARCH__) + int tid = blockIdx.x * blockDim.x + threadIdx.x; + data[tid] = data[tid] * scale; +#endif +} + int QkvToContextPluginDynamic::enqueue( const nvinfer1::PluginTensorDesc *input_desc, const nvinfer1::PluginTensorDesc *output_desc, const void *const *inputs, @@ -291,10 +299,17 @@ int QkvToContextPluginDynamic::enqueue( platform::DeviceContextPool::Instance().Get( platform::CUDAPlace(device_id))); + int n_q = seq_len * head_number_ * head_size_; + constexpr int threads = 128; + int blocks = (n_q + threads - 1) / threads; + + apply_scale<<>>(tptr, static_cast(scale_), + n_q); + const platform::CUDADeviceContext &dev_ctx = *device_ctx; operators::math::MultiHeadGPUComputeFunctor multihead_compute_func; multihead_compute_func(dev_ctx, batch, seq_len, head_number_, head_size_, - qkptr, input1_data, tptr, half(scale_), half(0.0)); + qkptr, input1_data, tptr, half(1.), half(0.0)); int grid = batch * head_number_ * seq_len; int block = head_size_; From 025132075612c5b2af4185f3963e834a1776950b Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Tue, 11 May 2021 11:10:56 +0800 Subject: [PATCH 052/156] fix find_unused_parameters default value (#32829) fix error log for reducer fix doc fix bug of utest fix spawn fix converage --- .../framework/distributed_strategy.proto | 2 +- paddle/fluid/imperative/reducer.cc | 110 ++++++++++-------- paddle/fluid/imperative/reducer.h | 8 +- .../fleet/base/distributed_strategy.py | 2 +- python/paddle/fluid/dygraph/parallel.py | 15 +-- .../parallel_dygraph_gradient_check.py | 4 +- .../tests/unittests/spawn_runner_base.py | 1 + .../fluid/tests/unittests/test_dist_base.py | 11 +- .../test_parallel_dygraph_control_flow.py | 6 + .../unittests/test_parallel_dygraph_mnist.py | 1 + 10 files changed, 95 insertions(+), 65 deletions(-) diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 654b88920acaf..99a6eb6b67472 100644 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -172,7 +172,7 @@ message DistributedStrategy { optional bool fp16_allreduce = 25 [ default = false ]; optional bool sharding = 26 [ default = false ]; optional float last_comm_group_size_MB = 27 [ default = 1 ]; - optional bool find_unused_parameters = 28 [ default = true ]; + optional bool find_unused_parameters = 28 [ default = false ]; optional bool tensor_parallel = 29 [ default = false ]; optional RecomputeConfig recompute_configs = 101; diff --git a/paddle/fluid/imperative/reducer.cc b/paddle/fluid/imperative/reducer.cc index e3dd0a2aa75b4..0f6676ed48f34 100644 --- a/paddle/fluid/imperative/reducer.cc +++ b/paddle/fluid/imperative/reducer.cc @@ -297,7 +297,7 @@ Reducer::Reducer(const std::vector> &vars, is_sparse_gradient_(is_sparse_gradient), parallel_ctx_(parallel_ctx), group_size_limits_(group_size_limits), - find_unused_vars_(find_unused_vars) { + find_unused_vars_each_step_(find_unused_vars) { VLOG(3) << "Start construct the Reducer ..."; nrings_ = parallel_ctx->GetNRings(); nranks_ = parallel_ctx->GetNRanks(); @@ -457,42 +457,8 @@ void Reducer::PrepareDeps(const std::unordered_set &init_nodes) { } } -// After each batch is calculated, the counter of each group(group.pending_) -// and allreudce sequence counter(next_group_) will be cleaned up again. -void Reducer::PrepareForBackward( +void Reducer::TraverseBackwardGraph( const std::vector> &outputs) { - VLOG(3) << "after forward, then reset count for backward."; - next_group_ = 0; - std::for_each(groups_.begin(), groups_.end(), [](Group &group) { - group.pending_ = group.variable_indices_.size(); - group.sparse_contents_ = nullptr; - }); - - // reinitialize vars_marked_ready_ for next iteration - vars_marked_ready_.clear(); - vars_marked_ready_.resize(vars_.size(), false); - - PADDLE_ENFORCE_EQ( - groups_need_finalize_, false, - platform::errors::PreconditionNotMet( - "A serious error has occurred here. There may be several reasons: " - "1) Please note that all forward outputs derived from the module " - "parameters must participate in the calculation of losses and " - "subsequent gradient calculations. If not, the wrapper will hang, " - "waiting for autograd to generate gradients for these parameters. " - "you can use detach or stop_gradient to make the unused parameters " - "detached from the autograd graph. " - "2) Used multiple forwards and one backward. You may be able to wrap " - "multiple forwards in a model.")); - - // The first var to trigger the unused parameter - has_marked_unused_vars_ = false; - unused_vars_.clear(); - - if (!find_unused_vars_) { - return; - } - node_deps_.clear(); std::queue> q; std::unordered_set var_visited; @@ -554,8 +520,50 @@ void Reducer::PrepareForBackward( << "] is not used"; } } +} - if (unused_vars_.empty()) { +// After each batch is calculated, the counter of each group(group.pending_) +// and allreudce sequence counter(next_group_) will be cleaned up again. +void Reducer::PrepareForBackward( + const std::vector> &outputs) { + VLOG(3) << "after forward, then reset count for backward."; + next_group_ = 0; + std::for_each(groups_.begin(), groups_.end(), [](Group &group) { + group.pending_ = group.variable_indices_.size(); + group.sparse_contents_ = nullptr; + }); + + // reinitialize vars_marked_ready_ for next iteration + vars_marked_ready_.clear(); + vars_marked_ready_.resize(vars_.size(), false); + + PADDLE_ENFORCE_EQ( + groups_need_finalize_, false, + platform::errors::PreconditionNotMet( + "A serious error has occurred here. Please " + "set find_unused_parameters=True to traverse backward graph " + "in each step to prepare reduce in advance. If you have " + "set, There may be several reasons for this error: " + "1) Please note that all forward outputs derived from the module " + "parameters must participate in the calculation of losses and " + "subsequent gradient calculations. If not, the wrapper will hang, " + "waiting for autograd to generate gradients for these parameters. " + "you can use detach or stop_gradient to make the unused parameters " + "detached from the autograd graph. " + "2) Used multiple forwards and one backward. You may be able to wrap " + "multiple forwards in a model.")); + + // The first var to trigger the unused parameter + has_marked_unused_vars_ = false; + + if (find_unused_vars_once_ || find_unused_vars_each_step_) { + unused_vars_.clear(); + TraverseBackwardGraph(outputs); + // only check once in first step + find_unused_vars_once_ = false; + } + + if (find_unused_vars_each_step_ && unused_vars_.empty()) { LOG_FIRST_N(WARNING, 1) << "All parameters are involved in the backward pass. " "It is recommended to set find_unused_parameters to False " @@ -564,7 +572,9 @@ void Reducer::PrepareForBackward( "will occur. Please make it clear that in the subsequent " "training, there will be no parameters that are not used " "in the backward pass, and then set find_unused_parameters"; - } else if (unused_vars_.size() == vars_.size()) { + } + + if (unused_vars_.size() == vars_.size()) { LOG_FIRST_N(WARNING, 1) << "There is no parameter in the device involved " "in the backward calculation. If there are " @@ -595,13 +605,13 @@ void Reducer::AddDistHook(size_t var_index) { local_used_vars_[var_index] = 1; - // rebuild group when find_unused_vars_ is false + // rebuild group when find_unused_vars_each_step_ is false if (NeedRebuildGroup()) { rebuild_vars_.push_back(vars_[var_index]); rebuild_var_indices_.push_back(var_index); } - if (!has_marked_unused_vars_ && find_unused_vars_) { + if (!has_marked_unused_vars_) { has_marked_unused_vars_ = true; for (const auto &unused_index : unused_vars_) { MarkVarReady(unused_index, false); @@ -622,7 +632,9 @@ void Reducer::MarkVarReady(const size_t var_index, const bool is_used_var) { if (vars_marked_ready_[var_index]) { auto error_info = string::Sprintf( "Error happened, when parameter[%d][%s] has been ready before. " - "There may be several reasons for this error: " + "Please set find_unused_parameters=True to traverse backward graph " + "in each step to prepare reduce in advance. If you have set, " + "there may be several reasons for this error: " "1) In multiple reentrant backward phase, some parameters are reused." "2) Using model parameters outside of forward function. Please " "make sure that model parameters are not shared in concurrent " @@ -690,10 +702,16 @@ void Reducer::MarkVarReady(const size_t var_index, const bool is_used_var) { } } else { // process sparse group - PADDLE_ENFORCE_EQ(HasGrad(var_index), true, - platform::errors::PreconditionNotMet( - "The sparse parameter[%d][%s] must have a gradient", - var_index, vars_[var_index]->Name())); + PADDLE_ENFORCE_EQ( + HasGrad(var_index), true, + platform::errors::PreconditionNotMet( + "The sparse parameter[%d][%s] should have gradient. " + "Currently, DataParallel does not support sparse " + "parameters without generating gradients during training. " + "For example, if is_sparese=True is used in Embedding, " + "the current step of this parameter cannot generate gradient " + "because of stop_gradient/detatch, where error will occur.", + var_index, vars_[var_index]->Name())); auto var_base = vars_[var_index]->GradVarBase(); // need to check tensor type PADDLE_ENFORCE_EQ( @@ -943,7 +961,7 @@ void Reducer::FinalizeBackward() { InitializeGroups(group_indices_); } - if (find_unused_vars_) { + if (find_unused_vars_each_step_) { // TODO(liuyuhui) support xpu about Tensorcopy/TensorFromVector/TensorToVector #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) ProcessUnusedDenseVars(); diff --git a/paddle/fluid/imperative/reducer.h b/paddle/fluid/imperative/reducer.h index 0d613dbea8963..8392ab2c704d5 100644 --- a/paddle/fluid/imperative/reducer.h +++ b/paddle/fluid/imperative/reducer.h @@ -162,13 +162,16 @@ class Reducer { std::vector> RebuildGruops(); inline bool NeedRebuildGroup() { - return !has_rebuilt_group_ && !find_unused_vars_; + return !has_rebuilt_group_ && !find_unused_vars_each_step_; } void ProcessUnusedDenseVars(); bool HasGrad(size_t var_index); + void TraverseBackwardGraph( + const std::vector>& outputs); + private: std::vector> vars_; std::vector> group_indices_; @@ -195,7 +198,8 @@ class Reducer { std::unordered_map var_index_map_; std::vector unused_vars_; bool has_marked_unused_vars_{false}; - bool find_unused_vars_{false}; + bool find_unused_vars_each_step_{false}; + bool find_unused_vars_once_{true}; bool groups_need_finalize_{false}; #ifdef PADDLE_WITH_XPU_BKCL // comm_pool_ is used for scheduling allreduce in multi Kunlun cards training. diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index 9fed3a8550c40..ab120898a7995 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -626,7 +626,7 @@ def find_unused_parameters(self): Indicating whether we are using find_unused_parameters to find unused parameters in DataParallel. - Default value: True + Default value: False Examples: diff --git a/python/paddle/fluid/dygraph/parallel.py b/python/paddle/fluid/dygraph/parallel.py index ca5e5606e432b..2be062962ec9d 100644 --- a/python/paddle/fluid/dygraph/parallel.py +++ b/python/paddle/fluid/dygraph/parallel.py @@ -417,14 +417,15 @@ class DataParallel(layers.Layer): Note that setting the find_unused_parameters to True will affect computing performance. Therefore, if all parameters are sure to participate in the loss calculation and the - autograd graph construction, please set it False. Default: True. + autograd graph construction, please set it False. Default: False. Returns: Layer: The data paralleled module. Examples: .. code-block:: python - + + # required: distributed import paddle import paddle.nn as nn import paddle.optimizer as opt @@ -474,7 +475,7 @@ def __init__(self, strategy=None, comm_buffer_size=25, last_comm_buffer_size=1, - find_unused_parameters=True): + find_unused_parameters=False): super(DataParallel, self).__init__(layers.full_name() + "_data_parallel") @@ -576,12 +577,8 @@ def _find_varbase(self, obj): def forward(self, *inputs, **kwargs): outputs = self._layers(*inputs, **kwargs) if self._strategy.nranks > 1 and framework._dygraph_tracer()._has_grad: - if self.find_unused_parameters: - self._reducer.prepare_for_backward( - list(self._find_varbase(outputs))) - else: - self._reducer.prepare_for_backward(list(self._find_varbase([]))) - + self._reducer.prepare_for_backward( + list(self._find_varbase(outputs))) return outputs @deprecated( diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py index 7002352240973..5c518976d1f36 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py @@ -74,8 +74,8 @@ def test_multiple_gpus(self): state_dict = model_a.state_dict() model_b.set_state_dict(state_dict) - model_a = paddle.DataParallel(model_a) - model_b = paddle.DataParallel(model_b) + model_a = paddle.DataParallel(model_a, find_unused_parameters=True) + model_b = paddle.DataParallel(model_b, find_unused_parameters=True) ones_input = paddle.ones(shape=(batch, in_dim)) ones_input.stop_gradient = True diff --git a/python/paddle/fluid/tests/unittests/spawn_runner_base.py b/python/paddle/fluid/tests/unittests/spawn_runner_base.py index 278d7b27c5288..2719e28fea08b 100644 --- a/python/paddle/fluid/tests/unittests/spawn_runner_base.py +++ b/python/paddle/fluid/tests/unittests/spawn_runner_base.py @@ -27,6 +27,7 @@ class SpawnAssistTestArgs(object): update_method = "local" trainer_id = 0 + find_unused_parameters = False class TestDistSpawnRunner(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index 37494294418f1..edc510e4e766d 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -548,7 +548,10 @@ def run_trainer_with_spawn(self, args): # 4. train model model, train_reader, opt = self.get_model() if args.update_method == "nccl2": - model = paddle.DataParallel(model) + if args.find_unused_parameters: + model = paddle.DataParallel(model, find_unused_parameters=True) + else: + model = paddle.DataParallel(model, find_unused_parameters=False) out_losses = [] for step_id, data in enumerate(train_reader()): @@ -581,8 +584,8 @@ def run_use_fleet_api_trainer(self, args): # set strategy strategy = fleet.DistributedStrategy() - if not args.find_unused_parameters: - strategy.find_unused_parameters = False + if args.find_unused_parameters: + strategy.find_unused_parameters = True # 3. init parallel env if args.update_method == "nccl2" or "bkcl": @@ -737,7 +740,7 @@ def setUp(self): self._save_model = False self._fuse_all_reduce = None self._accumulate_gradient = False - self._find_unused_parameters = True + self._find_unused_parameters = False self._setup_config() global DIST_UT_PORT diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_control_flow.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_control_flow.py index fa571bde5e43b..3c45b2c795037 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_control_flow.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_control_flow.py @@ -30,6 +30,7 @@ def _setup_config(self): self._sync_mode = False self._nccl2_mode = True self._dygraph = True + self._find_unused_parameters = True def test_net(self): if fluid.core.is_compiled_with_cuda(): @@ -46,6 +47,7 @@ def _setup_config(self): self._nccl2_mode = True self._dygraph = True self._use_fleet_api = True + self._find_unused_parameters = True class TestFleetDygraphControlFlowSameAccGrad(TestDygraphControlFlowSame): @@ -54,6 +56,7 @@ def _setup_config(self): self._nccl2_mode = True self._dygraph = True self._accumulate_gradient = True + self._find_unused_parameters = True class TestDygraphControlFlowDiff(TestDistBase): @@ -61,6 +64,7 @@ def _setup_config(self): self._sync_mode = False self._nccl2_mode = True self._dygraph = True + self._find_unused_parameters = True def test_net(self): if fluid.core.is_compiled_with_cuda(): @@ -77,6 +81,7 @@ def _setup_config(self): self._nccl2_mode = True self._dygraph = True self._use_fleet_api = True + self._find_unused_parameters = True class TestFleetDygraphControlFlowDiffAccGrad(TestDygraphControlFlowDiff): @@ -85,6 +90,7 @@ def _setup_config(self): self._nccl2_mode = True self._dygraph = True self._accumulate_gradient = True + self._find_unused_parameters = True if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_mnist.py index 782d2304619f2..0c55e135721ce 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_mnist.py @@ -31,6 +31,7 @@ def _setup_config(self): self._sync_mode = False self._nccl2_mode = True self._dygraph = True + self._find_unused_parameters = True def test_mnist(self): if fluid.core.is_compiled_with_cuda(): From 4ccd9a0a86ad550a861c954d70e28ef15741b310 Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Wed, 12 May 2021 23:09:32 +0800 Subject: [PATCH 053/156] fix dataloader exit hang when join re-enter (#32835) * fix dataloader exit hang when join re-enter. test=develop * double check _shutdown. test=develop --- .../fluid/dataloader/dataloader_iter.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index 52ab83698592a..1f928bfc8a689 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -289,10 +289,14 @@ def __init__(self, loader): # if user exit python program when dataloader is still # iterating, resource may no release safely, so we - # add __del__ function to to CleanupFuncRegistrar - # to make sure __del__ is always called when program + # add _shutdown_on_exit function to to CleanupFuncRegistrar + # to make sure _try_shutdown_all is always called when program # exit for resoure releasing safely - CleanupFuncRegistrar.register(self.__del__) + # worker join may hang for in _try_shutdown_all call in atexit + # for main process is in atexit state in some OS, so we add + # timeout=1 for shutdown function call in atexit, for shutdown + # function call in __del__, we keep it as it is + CleanupFuncRegistrar.register(self._shutdown_on_exit) def _init_workers(self): # multiprocess worker and indice queue list initial as empty @@ -363,7 +367,7 @@ def _shutdown_worker(self, worker_id): self._indices_queues[worker_id].put(None) self._worker_status[worker_id] = False - def _try_shutdown_all(self): + def _try_shutdown_all(self, timeout=None): if not self._shutdown: try: self._exit_thread_expectedly() @@ -376,11 +380,12 @@ def _try_shutdown_all(self): for i in range(self._num_workers): self._shutdown_worker(i) - for w in self._workers: - w.join() - for q in self._indices_queues: - q.cancel_join_thread() - q.close() + if not self._shutdown: + for w in self._workers: + w.join(timeout) + for q in self._indices_queues: + q.cancel_join_thread() + q.close() finally: core._erase_process_pids(id(self)) self._shutdown = True @@ -560,6 +565,9 @@ def _try_put_indices(self): def __del__(self): self._try_shutdown_all() + def _shutdown_on_exit(self): + self._try_shutdown_all(1) + def __next__(self): try: # _batches_outstanding here record the total batch data number From 4831e378655fda3d928026b10e98ee423dd6be11 Mon Sep 17 00:00:00 2001 From: cc <52520497+juncaipeng@users.noreply.github.com> Date: Mon, 17 May 2021 16:45:42 +0800 Subject: [PATCH 054/156] fix the error of fake_quant_dequant op name (#32866) (#32879) --- .../paddle/fluid/contrib/slim/quantization/imperative/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py b/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py index 004e1c1aa9bc5..491f8a7e25cbc 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py @@ -39,7 +39,7 @@ fake_quantize_dequantize_types = [ "fake_quantize_dequantize_abs_max", - "fake_quantize_dequantize_channel_wise_abs_max", + "fake_channel_wise_quantize_dequantize_abs_max", "fake_quantize_dequantize_moving_average_abs_max" ] From b619648c0d53eae3846867b880c410456a8d285b Mon Sep 17 00:00:00 2001 From: houj04 <35131887+houj04@users.noreply.github.com> Date: Tue, 18 May 2021 10:21:48 +0800 Subject: [PATCH 055/156] bugfix: parallel_executor for xpu should use BindThreadedSSAGraphExecutor (#32792) (#32933) --- paddle/fluid/framework/parallel_executor.cc | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index 73a699b41c8e0..eb021609e8258 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -1407,10 +1407,23 @@ std::vector ParallelExecutor::CreateSSAGraphExecutor( exec_strategy, member_->local_scopes_, member_->local_exec_scopes_, member_->places_, graph)); } else { - VLOG(3) << "use FastThreadedSSAGraphExecutor"; - member_->executor_.reset(new details::FastThreadedSSAGraphExecutor( - exec_strategy, member_->local_scopes_, member_->local_exec_scopes_, - member_->places_, graph)); + if (member_->use_device_ == p::kXPU) { +#if defined(PADDLE_WITH_XPU) + VLOG(3) << "use BindThreadedSSAGraphExecutor"; + member_->executor_.reset(new details::BindThreadedSSAGraphExecutor( + exec_strategy, member_->local_scopes_, + member_->local_exec_scopes_, member_->places_, graph)); +#else + PADDLE_THROW(platform::errors::PermissionDenied( + "Paddle can't use XPU device since it's not compiled with XPU," + "Please recompile or reinstall Paddle with XPU support.")); +#endif + } else { + VLOG(3) << "use FastThreadedSSAGraphExecutor"; + member_->executor_.reset(new details::FastThreadedSSAGraphExecutor( + exec_strategy, member_->local_scopes_, + member_->local_exec_scopes_, member_->places_, graph)); + } } final_graphs.emplace_back(graph); } From 7b0b064da4a8c5f5fa5935ec72d52d0b27657580 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Tue, 18 May 2021 10:59:02 +0800 Subject: [PATCH 056/156] [cherry-pick] Fix CI Python3 on release/2.1 (#32930) Fix CI Python3 on release/2.1 #32930 --- python/unittest_py/requirements.txt | 1 + tools/check_op_desc.py | 8 +++----- tools/summary_env.py | 5 +++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt index 5a59935887bbe..752f3545c69cc 100644 --- a/python/unittest_py/requirements.txt +++ b/python/unittest_py/requirements.txt @@ -10,3 +10,4 @@ scipy>=0.19.0, <=1.2.1 ; python_version<"3.5" scipy<=1.3.1 ; python_version=="3.5" scipy ; python_version>"3.5" prettytable +distro diff --git a/tools/check_op_desc.py b/tools/check_op_desc.py index 15e410401216c..78abb6f36c606 100644 --- a/tools/check_op_desc.py +++ b/tools/check_op_desc.py @@ -17,8 +17,6 @@ from paddle.utils import OpLastCheckpointChecker from paddle.fluid.core import OpUpdateType -SAME = 0 - INPUTS = "Inputs" OUTPUTS = "Outputs" ATTRS = "Attrs" @@ -71,7 +69,7 @@ def diff_vars(origin_vars, new_vars): vars_name_only_in_new = set(new_vars.keys()) - set(origin_vars.keys()) for var_name in common_vars_name: - if cmp(origin_vars.get(var_name), new_vars.get(var_name)) == SAME: + if origin_vars.get(var_name) == new_vars.get(var_name): continue else: error, var_error = True, True @@ -120,7 +118,7 @@ def diff_attr(ori_attrs, new_attrs): attrs_only_in_new = set(new_attrs.keys()) - set(ori_attrs.keys()) for attr_name in common_attrs: - if cmp(ori_attrs.get(attr_name), new_attrs.get(attr_name)) == SAME: + if ori_attrs.get(attr_name) == new_attrs.get(attr_name): continue else: error, attr_error = True, True @@ -184,7 +182,7 @@ def compare_op_desc(origin_op_desc, new_op_desc): new = json.loads(new_op_desc) desc_error_message = {} version_error_message = {} - if cmp(origin_op_desc, new_op_desc) == SAME: + if origin_op_desc == new_op_desc: return desc_error_message, version_error_message for op_type in origin: diff --git a/tools/summary_env.py b/tools/summary_env.py index 38bae87651d4b..d12e644cc28da 100644 --- a/tools/summary_env.py +++ b/tools/summary_env.py @@ -13,6 +13,7 @@ # limitations under the License. import os import sys +import distro import platform import subprocess @@ -47,8 +48,8 @@ def get_os_info(): plat = "macOs" ver = platform.mac_ver()[0] elif platform.system() == "Linux": - plat = platform.linux_distribution()[0] - ver = platform.linux_distribution()[1] + plat = distro.linux_distribution()[0] + ver = distro.linux_distribution()[1] elif platform.system() == "Windows": plat = "Windows" ver = platform.win32_ver()[0] From 4639f5de0c5dc5eee80c381c88ec81483a1fd432 Mon Sep 17 00:00:00 2001 From: WeiXin Date: Tue, 18 May 2021 14:07:21 +0800 Subject: [PATCH 057/156] [Cherry-pick]Add code examples for paddle.save/load (#32900) (#32929) * doc of paddle.save/load * polish doc of paddle.save/load --- python/paddle/framework/io.py | 61 +++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 493574c5bef47..de2116cd4382d 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -496,7 +496,7 @@ def save(obj, path, protocol=2, **configs): Save an object to the specified path. .. note:: - Now supports saving ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor. + Now supports saving ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor, Program. .. note:: Different from ``paddle.jit.save``, since the save result of ``paddle.save`` is a single file, @@ -544,7 +544,18 @@ def save(obj, path, protocol=2, **configs): # save weight of emb paddle.save(emb.weight, "emb.weight.pdtensor") - # example 2: static graph + # example 2: Save multiple state_dict at the same time + from paddle import nn + from paddle.optimizer import Adam + + layer = paddle.nn.Linear(3, 4) + adam = Adam(learning_rate=0.001, parameters=layer.parameters()) + obj = {'model': layer.state_dict(), 'opt': adam.state_dict(), 'epoch': 100} + path = 'example/model.pdparams' + paddle.save(obj, path) + + + # example 3: static graph import paddle import paddle.static as static @@ -570,6 +581,18 @@ def save(obj, path, protocol=2, **configs): # save/load state_dict path_state_dict = 'temp/model.pdparams' paddle.save(prog.state_dict("param"), path_tensor) + + # example 4: save program + import paddle + + paddle.enable_static() + + data = paddle.static.data( + name='x_static_save', shape=(None, 224), dtype='float32') + y_static = z = paddle.static.nn.fc(data, 10) + main_program = paddle.static.default_main_program() + path = "example/main_program.pdmodel" + paddle.save(main_program, path) ''' # 1. input check filename = os.path.basename(path) @@ -667,7 +690,7 @@ def load(path, **configs): Load an object can be used in paddle from specified path. .. note:: - Now supports loading ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor. + Now supports loading ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor, Program. .. note:: In order to use the model parameters saved by paddle more efficiently, @@ -714,8 +737,6 @@ def load(path, **configs): Examples: .. code-block:: python - import paddle - # example 1: dynamic graph import paddle emb = paddle.nn.Embedding(10, 10) @@ -744,7 +765,19 @@ def load(path, **configs): load_weight = paddle.load("emb.weight.pdtensor") - # example 2: static graph + # example 2: Load multiple state_dict at the same time + from paddle import nn + from paddle.optimizer import Adam + + layer = paddle.nn.Linear(3, 4) + adam = Adam(learning_rate=0.001, parameters=layer.parameters()) + obj = {'model': layer.state_dict(), 'opt': adam.state_dict(), 'epoch': 100} + path = 'example/model.pdparams' + paddle.save(obj, path) + obj_load = paddle.load(path) + + + # example 3: static graph import paddle import paddle.static as static @@ -773,6 +806,22 @@ def load(path, **configs): paddle.save(prog.state_dict("param"), path_tensor) load_state_dict = paddle.load(path_tensor) + + # example 4: load program + import paddle + + paddle.enable_static() + + data = paddle.static.data( + name='x_static_save', shape=(None, 224), dtype='float32') + y_static = z = paddle.static.nn.fc(data, 10) + main_program = paddle.static.default_main_program() + path = "example/main_program.pdmodel" + paddle.save(main_program, path) + load_main = paddle.load(path) + print(load_main) + + ''' if os.path.isfile(path): From ab1a4df95ca85884dd6c8c0ae012cddbf2c681d0 Mon Sep 17 00:00:00 2001 From: danleifeng <52735331+danleifeng@users.noreply.github.com> Date: Wed, 19 May 2021 11:40:25 +0800 Subject: [PATCH 058/156] =?UTF-8?q?=E3=80=90cherrypick=E3=80=91support=20c?= =?UTF-8?q?uda11=20for=20heterps;=20add=20profiler=20in=20oneps=20(#32957)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cherrypick for #32640 :add profile and fix dataset hang in heterps;test=develop * cherrypick for #32640 :add profile and fix dataset hang in heterps;test=develop * cherrypick for #32640 :add profile and fix dataset hang in heterps;test=develop --- paddle/fluid/framework/device_worker.h | 3 +++ paddle/fluid/framework/hogwild_worker.cc | 24 +++++++++++++++++++ .../distributed/fleet/dataset/dataset.py | 19 +++++++++++++-- python/paddle/fluid/dataset.py | 19 +++++++++++++-- python/paddle/fluid/executor.py | 3 +++ .../unittests/test_communicator_ps_gpu.py | 1 + 6 files changed, 65 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/framework/device_worker.h b/paddle/fluid/framework/device_worker.h index a49e492e48028..d33809a0a2b7c 100644 --- a/paddle/fluid/framework/device_worker.h +++ b/paddle/fluid/framework/device_worker.h @@ -266,6 +266,9 @@ class HogwildWorker : public CPUWorkerBase { HogwildWorkerParameter param_; std::vector skip_ops_; std::map stat_var_name_map_; +#ifdef PADDLE_WITH_HETERPS + platform::DeviceContext* dev_ctx_ = nullptr; +#endif }; class DownpourWorker : public HogwildWorker { diff --git a/paddle/fluid/framework/hogwild_worker.cc b/paddle/fluid/framework/hogwild_worker.cc index 89dc5c7d3ea93..b2d170888e28f 100644 --- a/paddle/fluid/framework/hogwild_worker.cc +++ b/paddle/fluid/framework/hogwild_worker.cc @@ -39,6 +39,9 @@ void HogwildWorker::Initialize(const TrainerDesc &desc) { for (int i = 0; i < param_.stat_var_names_size(); ++i) { stat_var_name_map_[param_.stat_var_names(i)] = 1; } +#ifdef PADDLE_WITH_HETERPS + dev_ctx_ = platform::DeviceContextPool::Instance().Get(place_); +#endif } void HogwildWorker::CreateThreadOperators(const ProgramDesc &program) { @@ -150,6 +153,9 @@ void HogwildWorker::TrainFilesWithProfiler() { VLOG(3) << "Going to run op " << op_name[i]; if (!need_skip) { ops_[i]->Run(*thread_scope_, place_); +#ifdef PADDLE_WITH_HETERPS + dev_ctx_->Wait(); +#endif } VLOG(3) << "Op " << op_name[i] << " Finished"; timeline.Pause(); @@ -167,6 +173,16 @@ void HogwildWorker::TrainFilesWithProfiler() { total_inst += cur_batch; ++batch_cnt; PrintFetchVars(); +#ifdef PADDLE_WITH_HETERPS + dev_ctx_->Wait(); + VLOG(1) << "GpuPs worker " << thread_id_ << " train cost " << total_time + << " seconds, ins_num: " << total_inst; + for (size_t i = 0; i < op_name.size(); ++i) { + VLOG(1) << "card:" << thread_id_ << ", op: " << op_name[i] + << ", mean time: " << op_total_time[i] / total_inst + << "s, totol time:" << op_total_time[i] << "sec"; + } +#else if (thread_id_ == 0) { if (batch_cnt > 0 && batch_cnt % 100 == 0) { for (size_t i = 0; i < ops_.size(); ++i) { @@ -178,6 +194,7 @@ void HogwildWorker::TrainFilesWithProfiler() { fprintf(stderr, "%6.2f instances/s\n", total_inst / total_time); } } +#endif thread_scope_->DropKids(); timeline.Start(); } @@ -195,7 +212,10 @@ void HogwildWorker::TrainFilesWithProfiler() { void HogwildWorker::TrainFiles() { platform::SetNumThreads(1); + platform::Timer timeline; + timeline.Start(); + int total_ins_num = 0; // how to accumulate fetched values here device_reader_->Start(); int cur_batch; @@ -213,9 +233,13 @@ void HogwildWorker::TrainFiles() { } } + total_ins_num += cur_batch; PrintFetchVars(); thread_scope_->DropKids(); } + timeline.Pause(); + VLOG(3) << "worker " << thread_id_ << " train cost " << timeline.ElapsedSec() + << " seconds, ins_num: " << total_ins_num; #if defined PADDLE_WITH_PSCORE if (thread_barrier_) { paddle::distributed::Communicator::GetInstance()->BarrierTriggerDecrement(); diff --git a/python/paddle/distributed/fleet/dataset/dataset.py b/python/paddle/distributed/fleet/dataset/dataset.py index 10c27ea91d249..e63369903190a 100644 --- a/python/paddle/distributed/fleet/dataset/dataset.py +++ b/python/paddle/distributed/fleet/dataset/dataset.py @@ -31,6 +31,7 @@ def __init__(self): self.dataset = core.Dataset("MultiSlotDataset") self.thread_num = 1 self.filelist = [] + self.use_ps_gpu = False def init(self, batch_size=1, @@ -212,6 +213,14 @@ def _prepare_to_run(self): self.dataset.set_data_feed_desc(self._desc()) self.dataset.create_readers() + def _set_use_ps_gpu(self, use_ps_gpu): + """ + set use_ps_gpu flag + Args: + use_ps_gpu: bool + """ + self.use_ps_gpu = use_ps_gpu + def _finish_to_run(self): self.dataset.destroy_readers() @@ -529,12 +538,18 @@ def _prepare_to_run(self): def _dynamic_adjust_before_train(self, thread_num): if not self.is_user_set_queue_num: - self.dataset.dynamic_adjust_channel_num(thread_num, False) + if self.use_ps_gpu: + self.dataset.dynamic_adjust_channel_num(thread_num, True) + else: + self.dataset.dynamic_adjust_channel_num(thread_num, False) self.dataset.dynamic_adjust_readers_num(thread_num) def _dynamic_adjust_after_train(self): if not self.is_user_set_queue_num: - self.dataset.dynamic_adjust_channel_num(self.thread_num, False) + if self.use_ps_gpu: + self.dataset.dynamic_adjust_channel_num(self.thread_num, True) + else: + self.dataset.dynamic_adjust_channel_num(self.thread_num, False) self.dataset.dynamic_adjust_readers_num(self.thread_num) def _set_queue_num(self, queue_num): diff --git a/python/paddle/fluid/dataset.py b/python/paddle/fluid/dataset.py index 86c63ababbbfd..db51cb549ad36 100644 --- a/python/paddle/fluid/dataset.py +++ b/python/paddle/fluid/dataset.py @@ -74,6 +74,7 @@ def __init__(self): self.dataset = core.Dataset("MultiSlotDataset") self.thread_num = 1 self.filelist = [] + self.use_ps_gpu = False def set_pipe_command(self, pipe_command): """ @@ -300,6 +301,14 @@ def _prepare_to_run(self): self.dataset.set_data_feed_desc(self.desc()) self.dataset.create_readers() + def _set_use_ps_gpu(self, use_ps_gpu): + """ + set use_ps_gpu flag + Args: + use_ps_gpu: bool + """ + self.use_ps_gpu = use_ps_gpu + def _finish_to_run(self): self.dataset.destroy_readers() @@ -391,7 +400,10 @@ def _prepare_to_run(self): ) def _dynamic_adjust_before_train(self, thread_num): if not self.is_user_set_queue_num: - self.dataset.dynamic_adjust_channel_num(thread_num, False) + if self.use_ps_gpu: + self.dataset.dynamic_adjust_channel_num(thread_num, True) + else: + self.dataset.dynamic_adjust_channel_num(thread_num, False) self.dataset.dynamic_adjust_readers_num(thread_num) @deprecated( @@ -400,7 +412,10 @@ def _dynamic_adjust_before_train(self, thread_num): ) def _dynamic_adjust_after_train(self): if not self.is_user_set_queue_num: - self.dataset.dynamic_adjust_channel_num(self.thread_num, False) + if self.use_ps_gpu: + self.dataset.dynamic_adjust_channel_num(self.thread_num, True) + else: + self.dataset.dynamic_adjust_channel_num(self.thread_num, False) self.dataset.dynamic_adjust_readers_num(self.thread_num) @deprecated( diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index 62a9c42ee0a61..620729795bc20 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -1507,6 +1507,9 @@ def _run_from_dataset(self, trainer._gen_trainer_desc() self._dump_debug_info(program=program, trainer=trainer) + # in case of calling _set_use_ps_gpu explicitly + if dataset.use_ps_gpu is False: + dataset._set_use_ps_gpu(trainer.proto_desc.use_ps_gpu) dataset._dynamic_adjust_before_train(trainer.proto_desc.thread_num) trainer_instance = self._default_executor.init_for_dataset( diff --git a/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py b/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py index 5de1ebf581372..0b956d5031fec 100644 --- a/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py @@ -73,6 +73,7 @@ def test_communicator_ps_gpu(self): dataset.init( batch_size=32, thread_num=1, pipe_command="cat", use_var=slots_vars) dataset.set_filelist(["test_communicator_ps_gpu.txt"]) + dataset._set_use_ps_gpu(1) dataset.load_into_memory() os.environ["TEST_MODE"] = "1" From b4b9438a2b370c9d18f3caebf36cf6ad074e7a71 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Wed, 19 May 2021 15:47:43 +0800 Subject: [PATCH 059/156] [Cherry-pick] add enforce check for set_value (#32972) (#32981) cherry-pick of #32972 --- paddle/fluid/pybind/imperative.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index 450c992d41118..231f7cfb1b5fb 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -710,6 +710,13 @@ void BindImperative(py::module *m_ptr) { imperative::NameVarBaseMap ins = {{"Input", {self}}}; imperative::NameVarBaseMap outs = {{"Out", {self}}}; + PADDLE_ENFORCE_EQ( + self->IsLeaf() && !self->OverridedStopGradient(), false, + platform::errors::InvalidArgument( + "Leaf Tensor (%s) that doesn't stop gradient can't use " + "inplace strategy.", + self->Name())); + auto value_tensor = value_obj.cast>(); ins.insert({"ValueTensor", {value_tensor}}); From bdce8a1dbd4efb817a3938ad29c080f041f322e6 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Wed, 19 May 2021 17:06:10 +0800 Subject: [PATCH 060/156] [Cherry-pick] Change Paddle CI-Cverage Python3.8 [32515] (#32960) Change Paddle CI-Cverage Python3.8 Cherry-pick 32515 --- paddle/scripts/paddle_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index b8b9f40aa33fc..0865d48c0d343 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1450,6 +1450,7 @@ function parallel_test() { mkdir -p ${PADDLE_ROOT}/build cd ${PADDLE_ROOT}/build pip install ${PADDLE_ROOT}/build/python/dist/*whl + cp ${PADDLE_ROOT}/build/python/paddle/fluid/tests/unittests/op_test.py ${PADDLE_ROOT}/build/python if [ "$WITH_GPU" == "ON" ] || [ "$WITH_ROCM" == "ON" ];then parallel_test_base_gpu else From c7848aca556d1984391edb35a212fdae41709e63 Mon Sep 17 00:00:00 2001 From: WeiXin Date: Thu, 20 May 2021 17:09:17 +0800 Subject: [PATCH 061/156] [Cherry-Pick]fix test_paddle_save_load and test_paddle_save_load_binary (#32949) (#33008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_paddle_save_load 单测随机挂：使用np.ndarray生成随机数组，可能生成nan，造成做对比时结果不匹配（nan != nan）。改为np.random.randn生成随机数组。 test_paddle_save_load_binary随机挂: 如果一个字符串不能解析为Program，windows上会有超时风险。解决方法：不在windows平台不加载'不能解析为Program的字符串'。 原始PR:#32949 --- .../fluid/tests/unittests/test_paddle_save_load.py | 9 ++++----- .../unittests/test_paddle_save_load_binary.py | 14 ++++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index 3a5c43b2bab3e..be2a6a653cc6f 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -412,11 +412,10 @@ def test_save_load_complex_object_dygraph_save(self): ] obj2 = {'k1': obj1, 'k2': state_dict, 'epoch': 123} obj3 = (paddle.randn( - [5, 4], dtype='float32'), np.ndarray( - [3, 4], dtype="float32"), { - "state_dict": state_dict, - "opt": state_dict - }) + [5, 4], dtype='float32'), np.random.randn(3, 4).astype("float32"), { + "state_dict": state_dict, + "opt": state_dict + }) obj4 = (np.random.randn(5, 6), (123, )) path1 = "test_save_load_any_complex_object_dygraph/obj1" diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py index 8b508d5c9ae79..7385da56beab3 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py @@ -19,6 +19,7 @@ import os import sys import six +import platform import paddle import paddle.nn as nn @@ -162,12 +163,13 @@ def test_save_load_lod_tensor(self): with self.assertRaises(NotImplementedError): path = 'test_save_load_error/temp' paddle.save({}, path, use_binary_format=True) - - with self.assertRaises(ValueError): - path = 'test_save_load_error/temp' - with open(path, "w") as f: - f.write('\0') - paddle.load(path) + # On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk. + if 'Windows' != platform.system(): + with self.assertRaises(ValueError): + path = 'test_save_load_error/temp' + with open(path, "w") as f: + f.write('\0') + paddle.load(path) with self.assertRaises(ValueError): temp_lod = fluid.core.LoDTensor() From ef2ee5e52ebf2ed8639b4279522676f6fff77929 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Thu, 20 May 2021 19:44:25 +0800 Subject: [PATCH 062/156] [cherry-pick] BugFix StaticAanlysis with gast.Subscript (#32969) (#32903) (#32986) * [Custom Op]Remove PADDLE_WITH_MKLDNN in custom_op (#32903) * [Dy2Stat]BugFix StaticAanlysis with gast.Subscript (#32969) * BugFix StaticAanlysis with gast.Subscript * remove codes --- .../dygraph_to_static/static_analysis.py | 3 ++ .../unittests/dygraph_to_static/test_list.py | 39 +++++++++++++++++++ .../utils/cpp_extension/extension_utils.py | 4 -- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py index 4b3b9fcf29885..cbe6b8a0ff942 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py @@ -368,5 +368,8 @@ def _get_node_var_type(self, cur_wrapper): if isinstance(node.func, gast.Name): return self.var_env.get_var_type(node.func.id) + if isinstance(node, gast.Subscript): + if self.is_tensor_node(node.value): + return {NodeVarType.TENSOR} return {NodeVarType.STATEMENT} diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py index 0243ef3a6ddae..e630c2b9c6feb 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py @@ -16,6 +16,7 @@ import unittest +import paddle import numpy as np import paddle.fluid as fluid from paddle.fluid.dygraph.jit import declarative @@ -61,6 +62,33 @@ def test_list_append_in_for_loop(x, iter_num): return a[0] +paddle.jit.set_code_level(100) + + +def test_list_append_in_for_subscript(x): + x = fluid.dygraph.to_variable(x) + iter_num = paddle.shape(x)[0] + a = [] + for i in range(iter_num): + x = x + 1 + a.append(x) + out = paddle.concat(a) + return out[0] + + +def test_list_append_in_while_loop_subscript(x): + x = fluid.dygraph.to_variable(x) + iter_num = paddle.shape(x)[0] + a = [] + i = 0 + while i < iter_num: + x = x + 1 + a.append(x) + i += 1 + out = paddle.concat(a) + return out[0] + + def test_list_append_in_for_loop_with_concat(x, iter_num): x = fluid.dygraph.to_variable(x) a = [] @@ -261,5 +289,16 @@ def init_dygraph_func(self): self.all_dygraph_funcs = [test_list_append_in_for_loop_with_concat, ] +class TestListInForLoopWithSubscript(TestListWithoutControlFlow): + def init_dygraph_func(self): + self.all_dygraph_funcs = [ + test_list_append_in_for_subscript, + test_list_append_in_while_loop_subscript + ] + + def init_data(self): + self.input = np.random.random((3, 4)).astype('float32') + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index ea46ea8b39195..104d979ef6785 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -469,10 +469,6 @@ def normalize_extension_kwargs(kwargs, use_cuda=False): ########################### -- END -- ########################### add_compile_flag(extra_compile_args, ['-w']) # disable warning - # Note(Aurelius84): This marco will impact memory layout of `Tensor`. - # We align it automatically with pre-installed Paddle. - if core.is_compiled_with_mkldnn(): - add_compile_flag(extra_compile_args, ['-DPADDLE_WITH_MKLDNN']) if use_cuda: extra_link_args.append('-lcudart') From 8ecaa8a5d8d7fb9a68e9b7a4677efb7fba3a7a34 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Thu, 20 May 2021 19:45:18 +0800 Subject: [PATCH 063/156] BugFix with ParseInputDataType from LodTensorArray (#32918) (#32984) * BugFix with ParseInputDataType from LodTensorArray * BugFix with ParseInputDataType from LodTensorArray --- paddle/fluid/framework/operator.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 955c917b2c1bf..1e26dab629016 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -1549,10 +1549,10 @@ void OperatorWithKernel::ParseInputDataType( } else if (var->IsType()) { t = &(var->Get().value()); } else if (var->IsType()) { - auto t_arr = var->Get(); - for (size_t j = 0; j < t_arr.size(); j++) { - if (t_arr[j].IsInitialized()) { - t = &(t_arr[j]); + auto t_arr = &var->Get(); + for (size_t j = 0; j < t_arr->size(); j++) { + if (t_arr->at(j).IsInitialized()) { + t = &(t_arr->at(j)); } } } From 26c29115af4003425f98d2a3ddc9fbf96c293b29 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Thu, 20 May 2021 19:45:54 +0800 Subject: [PATCH 064/156] [Cherry-pick]Refactor param_guard logic of @to_static (#32867) (#32859) (#32985) * [Dy2Static]Add param_guard in ParameterList to support @to_static * [Dy2Static] Refactor param_guard logic of @to_static (#32867) * Add param_guard in ParameterList to support @to_static * Refactor param_guard of @to_static * fix unittest failed * add more unittest --- python/paddle/fluid/dygraph/base.py | 61 ++++--- python/paddle/fluid/dygraph/container.py | 7 +- python/paddle/fluid/dygraph/layers.py | 4 + .../fluid/dygraph/varbase_patch_methods.py | 4 +- python/paddle/fluid/framework.py | 22 ++- python/paddle/fluid/layers/tensor.py | 8 +- .../dygraph_to_static/test_param_guard.py | 171 ++++++++++++++++++ 7 files changed, 242 insertions(+), 35 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py index be5d9ac58311b..c8e1370e44772 100644 --- a/python/paddle/fluid/dygraph/base.py +++ b/python/paddle/fluid/dygraph/base.py @@ -63,37 +63,52 @@ def program_desc_tracing_guard(enable): @signature_safe_contextmanager def param_guard(parameters): + from paddle.fluid.dygraph.dygraph_to_static.program_translator import in_declarative_mode # Note: parameters is a reference of self._parameters or self._buffers - if not framework.in_dygraph_mode() and parameters: + if in_declarative_mode() and not framework.in_dygraph_mode() and parameters: origin_parameters = parameters.copy() for name, var_base in parameters.items(): - if isinstance(var_base, core.VarBase): - # Convert ParamBase into Parameter with same attributes in dy2stat. - if isinstance(var_base, framework.ParamBase): - new_var = var_base._to_static_var(to_parameter=True) - else: - # Check whether has been created before. - if var_base.name in var_base.block.vars: - new_var = var_base.block.vars[var_base.name] - # Note(Aurelius84): Convert VarBase in self._buffers into Variabe with - # same attributes and set persistable=True to allow saving this var. - # Because users can create a VarBase in `__init__` like a - # `mask` Tensor or `hidden_0` in RNN layers, which is equivalent to a Parameter - # and necessary for inferring. It will be pruned if it's not necessary for inferring. - else: - # But if its shape is empty while created from `create_variable()`, we consider this buffer - # non-persistable. See case of `drop_state` in lstm api. - is_persistable = len(var_base.shape) > 0 - - new_var = var_base._to_static_var( - to_parameter=False, persistable=is_persistable) - parameters[name] = new_var + if isinstance(var_base, list): + new_var = [_convert_into_variable(var) for var in var_base] + else: + new_var = _convert_into_variable(var_base) + parameters[name] = new_var yield parameters.update(origin_parameters) else: yield +def _convert_into_variable(var_base): + """ + Convert Varbase into Variable. + """ + if isinstance(var_base, core.VarBase): + # Check whether has been created before. + new_var = var_base.block._find_var_recursive(var_base.name) + if new_var is not None: + assert isinstance(new_var, framework.Variable) + # Convert ParamBase into Parameter with same attributes in dy2stat. + elif isinstance(var_base, framework.ParamBase): + new_var = var_base._to_static_var(to_parameter=True) + else: + # Note(Aurelius84): Convert VarBase in self._buffers into Variable with + # same attributes and set persistable=True to allow saving this var. + # Because users can create a VarBase in `__init__` like a + # `mask` Tensor or `hidden_0` in RNN layers, which is equivalent to a Parameter + # and necessary for inferring. It will be pruned if it's not necessary for inferring. + + # But if its shape is empty while created from `create_variable()`, we consider this buffer + # non-persistable. See case of `drop_state` in lstm api. + is_persistable = len(var_base.shape) > 0 + + new_var = var_base._to_static_var( + to_parameter=False, persistable=is_persistable) + return new_var + else: + return var_base + + def enabled(): """ This function checks whether the program runs in dynamic graph mode or not. @@ -664,7 +679,7 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): if isinstance(framework._current_expected_place(), framework.core.CPUPlace): #TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. - # (1): eigen requires 16-bytes alignments, but the data of numpy array may not statisfy. + # (1): eigen requires 16-bytes alignments, but the data of numpy array may not statisfy. # Details: https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html # (2): when used in flask framework, it may result in hang. # Details: https://github.com/PaddlePaddle/Paddle/issues/26635 diff --git a/python/paddle/fluid/dygraph/container.py b/python/paddle/fluid/dygraph/container.py index c7ea412fec1b7..2938516e5bc44 100644 --- a/python/paddle/fluid/dygraph/container.py +++ b/python/paddle/fluid/dygraph/container.py @@ -15,6 +15,7 @@ from collections import OrderedDict from ..framework import Parameter from .layers import Layer +from .base import param_guard __all__ = [ 'Sequential', @@ -159,7 +160,8 @@ def __init__(self, parameters=None): self.add_parameter(str(idx), param) def __getitem__(self, idx): - return self._parameters[str(idx)] + with param_guard(self._parameters): + return self._parameters[str(idx)] def __setitem__(self, idx, param): assert isinstance(param, Parameter) @@ -169,7 +171,8 @@ def __len__(self): return len(self._parameters) def __iter__(self): - return iter(self._parameters.values()) + with param_guard(self._parameters): + return iter(self._parameters.values()) def append(self, parameter): """Appends a given parameter at the end of the list. diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index 18dfff434a2aa..542d13aa09aed 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -873,6 +873,10 @@ def _build_once(self, *args, **kwargs): pass def __call__(self, *inputs, **kwargs): + # NOTE(Aurelius84): Why we still need param_guard here? + # In case of ControlFlow, true_fn and false_fn will contain + # parameters that may not trigger logic of `Operator` to create + # them. we add this to make sure all parameters is available. with param_guard(self._parameters), param_guard(self._buffers): for forward_pre_hook in self._forward_pre_hooks.values(): hook_result = forward_pre_hook(self, inputs) diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index 37900b7880a35..644e25ab9183b 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -86,7 +86,7 @@ def _to_static_var(self, to_parameter=False, **kwargs): """ - # Note: getattr(self, attr, None) will call x.grad=x.gradient(), but gradient() only available in dygraph. + # Note: getattr(self, attr, None) will call x.grad=x.gradient(), but gradient() only available in dygraph. # It will fail. So, for propery in dygraph only, should not let it getattr(self, attr, None). attr_not_need_keys = ['grad'] if isinstance(self, ParamBase): @@ -108,6 +108,8 @@ def _to_static_var(self, to_parameter=False, **kwargs): if to_parameter or isinstance(self, ParamBase): del attr_kwargs['persistable'] + # NOTE(Aurelius84): All parameters should be placed into global block. + attr_kwargs['block'] = attr_kwargs['block'].program.global_block() static_var = Parameter(**attr_kwargs) else: static_var = Variable(**attr_kwargs) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 0e9d756848af4..3ca16b6667525 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -3222,14 +3222,22 @@ def append_op(self, *args, **kwargs): if attrs else {}, kwargs.get("stop_gradient", False)) else: + from paddle.fluid.dygraph.base import param_guard + op_desc = self.desc.append_op() - op = Operator( - block=self, - desc=op_desc, - type=kwargs.get("type", None), - inputs=kwargs.get("inputs", None), - outputs=kwargs.get("outputs", None), - attrs=kwargs.get("attrs", None)) + # NOTE(Aurelius84): In case of @to_static, all VarBase(s) should + # be converted into Variable(s) with same name and block location. + # This is ONE and ONLY logic of type transformation of dy2static. + inputs = kwargs.get("inputs", None) + outputs = kwargs.get("outputs", None) + with param_guard(inputs), param_guard(outputs): + op = Operator( + block=self, + desc=op_desc, + type=kwargs.get("type", None), + inputs=inputs, + outputs=outputs, + attrs=kwargs.get("attrs", None)) self.ops.append(op) diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index c0c07f593a3ed..987918493d3b4 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -580,8 +580,12 @@ def assign(input, output=None): input = numpy.array([input]) elif isinstance(input, (list, tuple)): input = numpy.array(input) - - if isinstance(input, Variable): + # NOTE(Aurelius84): Why we judge core.VarBase? + # In case of @to_static, a VarBase can be as input of `assign`, + # but in_dygraph_mode()==False under @to_static, which means + # isinstance(VarBase, Variable) == False. It will cause return None + # after this api. + if isinstance(input, (Variable, core.VarBase)): check_dtype(input.dtype, 'input', [ 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', 'bool' ], 'assign', '(When the type of input in assign is Variable.)') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py new file mode 100644 index 0000000000000..cd3c76412feac --- /dev/null +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py @@ -0,0 +1,171 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import numpy as np +import unittest + +from paddle.jit import to_static, ProgramTranslator + + +class NetWithParameterList(paddle.nn.Layer): + def __init__(self, in_size, out_size): + super(NetWithParameterList, self).__init__() + weight = self.create_parameter([in_size, out_size]) + bias = self.create_parameter([out_size], is_bias=True) + self.params = paddle.nn.ParameterList([weight, bias]) + + @to_static + def forward(self, x): + out = paddle.matmul(x, self.params[0]) + out = paddle.add(out, self.params[1]) + out = paddle.tanh(out) + return out + + +class NetWithParameterListIter(NetWithParameterList): + def __init__(self, in_size, out_size): + super(NetWithParameterListIter, self).__init__(in_size, out_size) + + @to_static + def forward(self, x): + # NOTE: manually trigger `__iter__` logic. + params = list(self.params.__iter__()) + out = paddle.matmul(x, params[0]) + out = paddle.add(out, params[1]) + out = paddle.tanh(out) + return out + + +class TestParameterList(unittest.TestCase): + def setUp(self): + self.seed = 2021 + self.iter_num = 5 + self.prog_trans = ProgramTranslator() + + def train(self, is_iter, to_static): + paddle.seed(self.seed) + np.random.seed(self.seed) + self.prog_trans.enable(to_static) + if is_iter: + net = NetWithParameterList(10, 3) + else: + net = NetWithParameterListIter(10, 3) + sgd = paddle.optimizer.SGD(0.1, parameters=net.parameters()) + + for batch_id in range(self.iter_num): + x = paddle.rand([4, 10], dtype='float32') + out = net(x) + loss = paddle.mean(out) + loss.backward() + sgd.step() + sgd.clear_grad() + + return loss + + def test_parameter_list(self): + static_loss = self.train(False, to_static=True) + dygraph_loss = self.train(False, to_static=False) + self.assertTrue( + np.allclose(dygraph_loss, static_loss), + msg='dygraph result is {}\nstatic result is {}'.format(dygraph_loss, + static_loss)) + + def test_parameter_list_iter(self): + static_loss = self.train(True, to_static=True) + dygraph_loss = self.train(True, to_static=False) + self.assertTrue( + np.allclose(dygraph_loss, static_loss), + msg='dygraph result is {}\nstatic result is {}'.format(dygraph_loss, + static_loss)) + + +class NetWithRawParamList(paddle.nn.Layer): + def __init__(self, in_size, out_size): + super(NetWithRawParamList, self).__init__() + weight = self.add_parameter('w', + self.create_parameter([in_size, out_size])) + bias = self.add_parameter( + 'b', self.create_parameter( + [out_size], is_bias=True)) + self.params = [weight] + self.bias_dict = {'b': bias} + + @to_static + def forward(self, x): + out = paddle.matmul(x, self.params[0]) + out = paddle.add(out, self.bias_dict['b']) + out = paddle.tanh(out) + return out + + +class TestRawParameterList(unittest.TestCase): + def setUp(self): + self.seed = 2021 + self.iter_num = 5 + self.prog_trans = ProgramTranslator() + + def init_net(self): + self.net = NetWithRawParamList(10, 3) + + def train(self, to_static): + paddle.seed(self.seed) + np.random.seed(self.seed) + self.prog_trans.enable(to_static) + self.init_net() + + sgd = paddle.optimizer.SGD(0.1, parameters=self.net.parameters()) + + for batch_id in range(self.iter_num): + x = paddle.rand([4, 10], dtype='float32') + out = self.net(x) + loss = paddle.mean(out) + loss.backward() + sgd.step() + sgd.clear_grad() + + return loss + + def test_parameter_list(self): + static_loss = self.train(to_static=True) + dygraph_loss = self.train(to_static=False) + self.assertTrue( + np.allclose(dygraph_loss, static_loss), + msg='dygraph result is {}\nstatic result is {}'.format(dygraph_loss, + static_loss)) + + +class NetWithSubLayerParamList(paddle.nn.Layer): + def __init__(self, sub_layer): + super(NetWithSubLayerParamList, self).__init__() + self.sub_layer = sub_layer + self.params = [sub_layer.weight] + self.bias_dict = {'b': sub_layer.bias} + + @to_static + def forward(self, x): + out = paddle.matmul(x, self.params[0]) + out = paddle.add(out, self.bias_dict['b']) + out = paddle.tanh(out) + return out + + +class TestSubLayerParameterList(TestRawParameterList): + def init_net(self): + fc = paddle.nn.Linear(10, 3) + self.net = NetWithSubLayerParamList(fc) + + +if __name__ == '__main__': + unittest.main() From 50356ebcdcc4d50cd02895086f460fb5d28ad7e0 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Fri, 21 May 2021 10:13:55 +0800 Subject: [PATCH 065/156] [Cherry-pick] Change Paddle CI-Cverage Python3.8 [32515] #33013 Change Paddle CI-Cverage Python3.8 Cherry-pick 32515 --- .../tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py | 1 - python/paddle/fluid/tests/unittests/test_fusion_gru_op.py | 4 ++-- python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py | 2 +- python/paddle/fluid/tests/unittests/test_gru_op.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py index c024ffbdb4b6a..7320efd259f45 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py @@ -19,7 +19,6 @@ import struct import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -from paddle.fluid.tests.unittests.op_test import OpTest from paddle.fluid.tests.unittests.test_fusion_gru_op import fusion_gru from paddle.fluid.tests.unittests.test_fusion_lstm_op import fc, ACTIVATION diff --git a/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py b/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py index 1e25b8034da0a..c241fc65d9b82 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py @@ -18,8 +18,8 @@ import numpy as np import math from op_test import OpTest -from test_gru_op import gru -from test_fusion_lstm_op import fc, ACTIVATION +from paddle.fluid.tests.unittests.test_gru_op import gru +from paddle.fluid.tests.unittests.test_fusion_lstm_op import fc, ACTIVATION def fusion_gru( diff --git a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py index 3928b6fa034ef..4899927a7694f 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py @@ -17,7 +17,7 @@ import unittest import numpy as np from op_test import OpTest -from test_lstm_op import lstm, ACTIVATION +from paddle.fluid.tests.unittests.test_lstm_op import lstm, ACTIVATION def fc(x, w, b): diff --git a/python/paddle/fluid/tests/unittests/test_gru_op.py b/python/paddle/fluid/tests/unittests/test_gru_op.py index 3ea47a5d690ea..3ec943ef2e04a 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_op.py @@ -19,7 +19,7 @@ import math import functools from op_test import OpTest -from test_lstm_op import ACTIVATION +from paddle.fluid.tests.unittests.test_lstm_op import ACTIVATION from paddle import fluid from paddle.fluid import Program, program_guard From 7c0b96e680e43dc972dbeadf55d872351c508aba Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Mon, 24 May 2021 18:14:16 +0800 Subject: [PATCH 066/156] update 2.0 public api in distributed (#32990) --- python/paddle/distributed/__init__.py | 94 +++++++++++-------- python/paddle/distributed/cloud_utils.py | 7 +- python/paddle/distributed/collective.py | 26 ++--- python/paddle/distributed/entry_attr.py | 2 +- python/paddle/distributed/fleet/__init__.py | 41 +++++--- .../paddle/distributed/fleet/ascend_utils.py | 2 + .../fleet/base/distributed_strategy.py | 2 +- .../distributed/fleet/base/fleet_base.py | 2 + .../fleet/base/meta_optimizer_factory.py | 2 + .../fleet/base/private_helper_function.py | 2 + .../distributed/fleet/base/role_maker.py | 2 + .../distributed/fleet/base/runtime_factory.py | 2 + .../fleet/base/strategy_compiler.py | 2 + .../distributed/fleet/base/util_factory.py | 3 +- .../paddle/distributed/fleet/cloud_utils.py | 2 + .../fleet/data_generator/__init__.py | 4 +- .../fleet/data_generator/data_generator.py | 2 + .../distributed/fleet/dataset/__init__.py | 10 +- .../distributed/fleet/dataset/dataset.py | 2 + .../fleet/dataset/index_dataset.py | 2 + python/paddle/distributed/fleet/launch.py | 2 + .../fleet/meta_optimizers/amp_optimizer.py | 2 + .../ascend/ascend_optimizer.py | 2 + .../meta_optimizers/ascend/ascend_parser.py | 2 + .../fleet/meta_optimizers/common.py | 2 + .../fleet/meta_optimizers/dgc_optimizer.py | 2 + .../dygraph_optimizer/__init__.py | 2 + .../hybrid_parallel_gradscaler.py | 2 + .../hybrid_parallel_optimizer.py | 2 + .../fp16_allreduce_optimizer.py | 2 + .../gradient_merge_optimizer.py | 2 + .../graph_execution_optimizer.py | 2 + .../fleet/meta_optimizers/lamb_optimizer.py | 2 + .../fleet/meta_optimizers/lars_optimizer.py | 2 + .../meta_optimizers/localsgd_optimizer.py | 2 + .../meta_optimizers/meta_optimizer_base.py | 2 + .../parameter_server_graph_optimizer.py | 2 + .../parameter_server_optimizer.py | 2 + .../meta_optimizers/pipeline_optimizer.py | 2 + .../meta_optimizers/recompute_optimizer.py | 2 + .../meta_optimizers/sharding/fp16_helper.py | 2 + .../sharding/gradient_clip_helper.py | 2 + .../sharding/offload_helper.py | 2 + .../fleet/meta_optimizers/sharding/prune.py | 2 + .../fleet/meta_optimizers/sharding/shard.py | 2 + .../sharding/weight_decay_helper.py | 2 + .../meta_optimizers/sharding_optimizer.py | 2 +- .../tensor_parallel_optimizer.py | 2 + .../fleet/meta_parallel/__init__.py | 15 ++- .../fleet/meta_parallel/meta_parallel_base.py | 2 + .../fleet/meta_parallel/model_parallel.py | 6 +- .../meta_parallel/parallel_layers/__init__.py | 13 ++- .../parallel_layers/mp_layers.py | 4 +- .../parallel_layers/pp_layers.py | 2 +- .../meta_parallel/parallel_layers/random.py | 5 +- .../fleet/meta_parallel/pipeline_parallel.py | 13 ++- .../fleet/meta_parallel/pp_utils/__init__.py | 4 +- .../fleet/meta_parallel/pp_utils/utils.py | 5 +- .../distributed/fleet/metrics/__init__.py | 20 ++-- .../distributed/fleet/metrics/metric.py | 2 + .../distributed/fleet/runtime/__init__.py | 2 + .../fleet/runtime/collective_runtime.py | 2 + .../fleet/runtime/parameter_server_runtime.py | 2 + .../distributed/fleet/runtime/the_one_ps.py | 2 + .../distributed/fleet/utils/__init__.py | 14 ++- python/paddle/distributed/fleet/utils/fs.py | 2 +- .../distributed/fleet/utils/http_server.py | 2 + .../fleet/utils/hybrid_parallel_util.py | 2 + .../distributed/fleet/utils/log_util.py | 2 + .../paddle/distributed/fleet/utils/ps_util.py | 2 + .../distributed/fleet/utils/recompute.py | 2 + python/paddle/distributed/launch.py | 2 + python/paddle/distributed/parallel.py | 9 +- python/paddle/distributed/spawn.py | 6 +- python/paddle/distributed/utils.py | 18 ++++ python/paddle/nn/__init__.py | 2 +- 76 files changed, 310 insertions(+), 119 deletions(-) diff --git a/python/paddle/distributed/__init__.py b/python/paddle/distributed/__init__.py index c882e94d2bade..47aa092fa9379 100644 --- a/python/paddle/distributed/__init__.py +++ b/python/paddle/distributed/__init__.py @@ -12,46 +12,60 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import spawn -from .spawn import spawn - -from . import parallel -from .parallel import init_parallel_env -from .parallel import get_rank -from .parallel import get_world_size -from paddle.fluid.dygraph.parallel import ParallelEnv #DEFINE_ALIAS -from paddle.distributed.fleet.dataset import * - -from . import collective -from .collective import * - -from .entry_attr import ProbabilityEntry -from .entry_attr import CountFilterEntry - -# start multiprocess apis -__all__ = ["spawn"] - -# dygraph parallel apis -__all__ += [ - "init_parallel_env", - "get_rank", - "get_world_size", - "ParallelEnv", - "InMemoryDataset", - "QueueDataset", -] +from .spawn import spawn # noqa: F401 -# dataset reader -__all__ += [ - "InMemoryDataset", - "QueueDataset", -] +from .parallel import init_parallel_env # noqa: F401 +from .parallel import get_rank # noqa: F401 +from .parallel import get_world_size # noqa: F401 -# entry for embedding -__all__ += [ - "ProbabilityEntry", - "CountFilterEntry", -] +from paddle.distributed.fleet.dataset import InMemoryDataset # noqa: F401 +from paddle.distributed.fleet.dataset import QueueDataset # noqa: F401 + +from .collective import broadcast # noqa: F401 +from .collective import all_reduce # noqa: F401 +from .collective import reduce # noqa: F401 +from .collective import all_gather # noqa: F401 +from .collective import scatter # noqa: F401 +from .collective import barrier # noqa: F401 +from .collective import ReduceOp # noqa: F401 +from .collective import split # noqa: F401 +from .collective import new_group # noqa: F401 +from .collective import recv # noqa: F401 +from .collective import get_group # noqa: F401 +from .collective import send # noqa: F401 +from .collective import wait # noqa: F401 + +from .fleet import BoxPSDataset # noqa: F401 -# collective apis -__all__ += collective.__all__ +from .entry_attr import ProbabilityEntry # noqa: F401 +from .entry_attr import CountFilterEntry # noqa: F401 + +from paddle.fluid.dygraph.parallel import ParallelEnv # noqa: F401 + +from . import cloud_utils # noqa: F401 +from . import utils # noqa: F401 + +__all__ = [ #noqa + "spawn", + "scatter", + "broadcast", + "ParallelEnv", + "new_group", + "init_parallel_env", + "QueueDataset", + "split", + "CountFilterEntry", + "get_world_size", + "get_group", + "all_gather", + "InMemoryDataset", + "barrier", + "all_reduce", + "send", + "reduce", + "recv", + "ReduceOp", + "wait", + "get_rank", + "ProbabilityEntry" +] diff --git a/python/paddle/distributed/cloud_utils.py b/python/paddle/distributed/cloud_utils.py index 962ba62b15f4a..34e55bf164673 100644 --- a/python/paddle/distributed/cloud_utils.py +++ b/python/paddle/distributed/cloud_utils.py @@ -14,7 +14,12 @@ import os import paddle -from paddle.distributed.utils import get_cluster, logger, get_gpus, get_cluster_from_args +from paddle.distributed.utils import get_cluster +from paddle.distributed.utils import logger +from paddle.distributed.utils import get_gpus +from paddle.distributed.utils import get_cluster_from_args + +__all__ = [] def get_cloud_cluster(args_node_ips, args_node_ip, args_port, selected_devices): diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index fefabaf69768e..85b8cafd6c315 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -15,8 +15,14 @@ import numpy as np import os from ..fluid.layer_helper import LayerHelper -from ..fluid.framework import Variable, OpProtoHolder, in_dygraph_mode, convert_np_dtype_to_dtype_ -from ..fluid.data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..fluid.framework import Variable +from ..fluid.framework import OpProtoHolder +from ..fluid.framework import in_dygraph_mode +from ..fluid.framework import convert_np_dtype_to_dtype_ +from ..fluid.data_feeder import convert_dtype +from ..fluid.data_feeder import check_variable_and_dtype +from ..fluid.data_feeder import check_type +from ..fluid.data_feeder import check_dtype from ..fluid.layers.tensor import fill_constant from ..fluid.layers import utils from ..fluid.dygraph.parallel import prepare_context @@ -25,21 +31,7 @@ import paddle.fluid as fluid import paddle.fluid.core as core -__all__ = [ - 'wait', - 'new_group', - 'get_group', - 'broadcast', - 'all_reduce', - 'reduce', - 'all_gather', - 'scatter', - 'barrier', - 'split', - 'ReduceOp', - 'send', - 'recv', -] +__all__ = [] class ReduceOp: diff --git a/python/paddle/distributed/entry_attr.py b/python/paddle/distributed/entry_attr.py index dbd899952af03..e219ef6434a3f 100644 --- a/python/paddle/distributed/entry_attr.py +++ b/python/paddle/distributed/entry_attr.py @@ -14,7 +14,7 @@ from __future__ import print_function -__all__ = ['ProbabilityEntry', 'CountFilterEntry'] +__all__ = [] class EntryAttr(object): diff --git a/python/paddle/distributed/fleet/__init__.py b/python/paddle/distributed/fleet/__init__.py index 403a02496afaa..5f9a61371d34f 100644 --- a/python/paddle/distributed/fleet/__init__.py +++ b/python/paddle/distributed/fleet/__init__.py @@ -13,21 +13,34 @@ # limitations under the License. # TODO: define distributed api under this directory, -from .base.role_maker import Role, UserDefinedRoleMaker, PaddleCloudRoleMaker -from .base.distributed_strategy import DistributedStrategy -from .base.fleet_base import Fleet -from .base.util_factory import UtilBase -from .dataset import * -from .data_generator import MultiSlotDataGenerator, MultiSlotStringDataGenerator -from . import metrics -from .base.topology import CommunicateTopology, HybridCommunicateGroup -from .meta_parallel import * +from .base.role_maker import Role # noqa: F401 +from .base.role_maker import UserDefinedRoleMaker # noqa: F401 +from .base.role_maker import PaddleCloudRoleMaker # noqa: F401 +from .base.distributed_strategy import DistributedStrategy # noqa: F401 +from .base.fleet_base import Fleet # noqa: F401 +from .base.util_factory import UtilBase # noqa: F401 +from .dataset import DatasetBase # noqa: F401 +from .dataset import InMemoryDataset # noqa: F401 +from .dataset import QueueDataset # noqa: F401 +from .dataset import FileInstantDataset # noqa: F401 +from .dataset import BoxPSDataset # noqa: F401 +from .data_generator.data_generator import MultiSlotDataGenerator # noqa: F401 +from .data_generator.data_generator import MultiSlotStringDataGenerator # noqa: F401 +from . import metrics # noqa: F401 +from .base.topology import CommunicateTopology +from .base.topology import HybridCommunicateGroup # noqa: F401 -__all__ = [ - "DistributedStrategy", "UtilBase", "UserDefinedRoleMaker", - "PaddleCloudRoleMaker", "Fleet", "MultiSlotDataGenerator", - "MultiSlotStringDataGenerator", "Role", "CommunicateTopology", - "HybridCommunicateGroup" +__all__ = [ #noqa + "CommunicateTopology", + "UtilBase", + "HybridCommunicateGroup", + "MultiSlotStringDataGenerator", + "UserDefinedRoleMaker", + "DistributedStrategy", + "Role", + "MultiSlotDataGenerator", + "PaddleCloudRoleMaker", + "Fleet" ] fleet = Fleet() diff --git a/python/paddle/distributed/fleet/ascend_utils.py b/python/paddle/distributed/fleet/ascend_utils.py index b64149f27bcac..708c76ac55abe 100644 --- a/python/paddle/distributed/fleet/ascend_utils.py +++ b/python/paddle/distributed/fleet/ascend_utils.py @@ -17,6 +17,8 @@ import paddle from paddle.distributed.fleet.launch_utils import get_cluster, logger, get_host_name_ip, DeviceMode +__all__ = [] + def _get_ascend_rankfile(rank_table_file_path): """ diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index ab120898a7995..25e571dba0c80 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -19,7 +19,7 @@ import google.protobuf.text_format import google.protobuf -__all__ = ["DistributedStrategy"] +__all__ = [] non_auto_func_called = True diff --git a/python/paddle/distributed/fleet/base/fleet_base.py b/python/paddle/distributed/fleet/base/fleet_base.py index 9e200f4ee5f6e..a7564a23a7cfb 100644 --- a/python/paddle/distributed/fleet/base/fleet_base.py +++ b/python/paddle/distributed/fleet/base/fleet_base.py @@ -33,6 +33,8 @@ from ..meta_optimizers import HybridParallelOptimizer from ..meta_optimizers import HybridParallelGradScaler +__all__ = [] + def _inited_runtime_handler_(func): def __impl__(*args, **kwargs): diff --git a/python/paddle/distributed/fleet/base/meta_optimizer_factory.py b/python/paddle/distributed/fleet/base/meta_optimizer_factory.py index 6989eec119f78..52eeebd0c126c 100755 --- a/python/paddle/distributed/fleet/base/meta_optimizer_factory.py +++ b/python/paddle/distributed/fleet/base/meta_optimizer_factory.py @@ -14,6 +14,8 @@ from ..meta_optimizers import * +__all__ = [] + meta_optimizer_names = list( filter(lambda name: name.endswith("Optimizer"), dir())) diff --git a/python/paddle/distributed/fleet/base/private_helper_function.py b/python/paddle/distributed/fleet/base/private_helper_function.py index 6af4a9e667528..c7ddd33d5d018 100644 --- a/python/paddle/distributed/fleet/base/private_helper_function.py +++ b/python/paddle/distributed/fleet/base/private_helper_function.py @@ -17,6 +17,8 @@ from contextlib import closing from six import string_types +__all__ = [] + def wait_server_ready(endpoints): """ diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py index 62c8faa0757c6..f89d73416960a 100644 --- a/python/paddle/distributed/fleet/base/role_maker.py +++ b/python/paddle/distributed/fleet/base/role_maker.py @@ -22,6 +22,8 @@ import paddle.fluid as fluid from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +__all__ = [] + class Role: WORKER = 1 diff --git a/python/paddle/distributed/fleet/base/runtime_factory.py b/python/paddle/distributed/fleet/base/runtime_factory.py index 9e612c6d530f1..85ff3e1e69c58 100644 --- a/python/paddle/distributed/fleet/base/runtime_factory.py +++ b/python/paddle/distributed/fleet/base/runtime_factory.py @@ -15,6 +15,8 @@ from ..runtime.parameter_server_runtime import ParameterServerRuntime from ..runtime.the_one_ps import TheOnePSRuntime +__all__ = [] + class RuntimeFactory(object): def __init__(self): diff --git a/python/paddle/distributed/fleet/base/strategy_compiler.py b/python/paddle/distributed/fleet/base/strategy_compiler.py index 7b146318abe62..b90e5b2bff7bf 100644 --- a/python/paddle/distributed/fleet/base/strategy_compiler.py +++ b/python/paddle/distributed/fleet/base/strategy_compiler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +__all__ = [] + def create_graph(optimizer_list): nsize = len(optimizer_list) diff --git a/python/paddle/distributed/fleet/base/util_factory.py b/python/paddle/distributed/fleet/base/util_factory.py index d982f14eaa5af..de101cd74c4e8 100644 --- a/python/paddle/distributed/fleet/base/util_factory.py +++ b/python/paddle/distributed/fleet/base/util_factory.py @@ -27,7 +27,8 @@ import subprocess import os import numpy as np -__all__ = ['UtilBase'] + +__all__ = [] class UtilFactory(object): diff --git a/python/paddle/distributed/fleet/cloud_utils.py b/python/paddle/distributed/fleet/cloud_utils.py index f5a24cf48ca06..0b1169e442263 100644 --- a/python/paddle/distributed/fleet/cloud_utils.py +++ b/python/paddle/distributed/fleet/cloud_utils.py @@ -16,6 +16,8 @@ import paddle from paddle.distributed.fleet.launch_utils import get_cluster, logger +__all__ = [] + def get_cloud_cluster(args_node_ips, device_mode, diff --git a/python/paddle/distributed/fleet/data_generator/__init__.py b/python/paddle/distributed/fleet/data_generator/__init__.py index 481df4064a4ec..230ada2abec06 100644 --- a/python/paddle/distributed/fleet/data_generator/__init__.py +++ b/python/paddle/distributed/fleet/data_generator/__init__.py @@ -11,4 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -from .data_generator import * +from .data_generator import DataGenerator # noqa: F401 + +__all__ = [] diff --git a/python/paddle/distributed/fleet/data_generator/data_generator.py b/python/paddle/distributed/fleet/data_generator/data_generator.py index 9d743fc38bf39..cceb81838c1d2 100644 --- a/python/paddle/distributed/fleet/data_generator/data_generator.py +++ b/python/paddle/distributed/fleet/data_generator/data_generator.py @@ -15,6 +15,8 @@ import os import sys +__all__ = [] + class DataGenerator(object): """ diff --git a/python/paddle/distributed/fleet/dataset/__init__.py b/python/paddle/distributed/fleet/dataset/__init__.py index 24b68596f2541..55b944abccd51 100644 --- a/python/paddle/distributed/fleet/dataset/__init__.py +++ b/python/paddle/distributed/fleet/dataset/__init__.py @@ -11,5 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -from .dataset import * -from .index_dataset import * +from .dataset import DatasetBase # noqa: F401 +from .dataset import InMemoryDataset # noqa: F401 +from .dataset import QueueDataset # noqa: F401 +from .dataset import FileInstantDataset # noqa: F401 +from .dataset import BoxPSDataset # noqa: F401 +from .index_dataset import TreeIndex # noqa: F401 + +__all__ = [] diff --git a/python/paddle/distributed/fleet/dataset/dataset.py b/python/paddle/distributed/fleet/dataset/dataset.py index e63369903190a..f8465a7fe5f7f 100644 --- a/python/paddle/distributed/fleet/dataset/dataset.py +++ b/python/paddle/distributed/fleet/dataset/dataset.py @@ -18,6 +18,8 @@ from google.protobuf import text_format import paddle.fluid.core as core +__all__ = [] + class DatasetBase(object): """ Base dataset class. """ diff --git a/python/paddle/distributed/fleet/dataset/index_dataset.py b/python/paddle/distributed/fleet/dataset/index_dataset.py index dfd3daa9570b9..c4c424fe2dc7e 100644 --- a/python/paddle/distributed/fleet/dataset/index_dataset.py +++ b/python/paddle/distributed/fleet/dataset/index_dataset.py @@ -13,6 +13,8 @@ # limitations under the License. from paddle.fluid import core +__all__ = [] + class Index(object): def __init__(self, name): diff --git a/python/paddle/distributed/fleet/launch.py b/python/paddle/distributed/fleet/launch.py index 69c5b325d182d..25b1013319178 100644 --- a/python/paddle/distributed/fleet/launch.py +++ b/python/paddle/distributed/fleet/launch.py @@ -75,6 +75,8 @@ import paddle.distributed.fleet.cloud_utils as cloud_utils import paddle.distributed.fleet.ascend_utils as ascend_utils +__all__ = [] + def _print_arguments(args): print("----------- Configuration Arguments -----------") diff --git a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py index 02505e01197dc..9ffb47789ee98 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py @@ -14,6 +14,8 @@ import paddle.fluid.contrib.mixed_precision as mixed_precision from .meta_optimizer_base import MetaOptimizerBase +__all__ = [] + class AMPOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py index 824225fd776d1..6282ac7b50983 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py @@ -24,6 +24,8 @@ HcomGroupConfig = namedtuple('HcomGroupConfig', ['name', 'nranks', 'rank_ids']) +__all__ = [] + class AscendIRParser(object): def __init__(self, auto_dp=False, world_rank_size=1): diff --git a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py index 19b5e910db299..3331a45b3d947 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py @@ -18,6 +18,8 @@ from paddle.distributed import fleet from functools import reduce +__all__ = [] + registerd_op = {## forwards "elementwise_add": "AddParser", "matmul": "MatMulParser", diff --git a/python/paddle/distributed/fleet/meta_optimizers/common.py b/python/paddle/distributed/fleet/meta_optimizers/common.py index 9e2723dad729a..707284a784c38 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/common.py +++ b/python/paddle/distributed/fleet/meta_optimizers/common.py @@ -19,6 +19,8 @@ from paddle.fluid import core, unique_name from ..base.private_helper_function import wait_server_ready +__all__ = [] + OpRole = core.op_proto_and_checker_maker.OpRole OP_ROLE_KEY = core.op_proto_and_checker_maker.kOpRoleAttrName() diff --git a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py index 7bd6832556933..b035f179317ac 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py @@ -15,6 +15,8 @@ from .meta_optimizer_base import MetaOptimizerBase import logging +__all__ = [] + class DGCOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/__init__.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/__init__.py index 4e41723cb622d..f0f26bd2e0d06 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/__init__.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/__init__.py @@ -12,3 +12,5 @@ # See the License for the specific language governing permissions and from .hybrid_parallel_optimizer import HybridParallelOptimizer from .hybrid_parallel_gradscaler import HybridParallelGradScaler + +__all__ = [] diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py index 13bb9d2acece2..d0e8034f5cae1 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py @@ -23,6 +23,8 @@ from paddle.fluid import core import paddle +__all__ = [] + class HybridParallelGradScaler: def __init__(self, scaler, hcg): diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py index 52e87173684a3..b7ac298d2223e 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py @@ -23,6 +23,8 @@ from paddle.fluid.framework import Variable from ...utils.log_util import logger +__all__ = [] + class HybridParallelClipGrad: def __init__(self, clip, hcg): diff --git a/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py index 411980ed01322..f636a31375785 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py @@ -14,6 +14,8 @@ from paddle.fluid import core, framework, unique_name from .meta_optimizer_base import MetaOptimizerBase +__all__ = [] + class FP16AllReduceOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py index 380fbc2e09ebf..949ef3e5f3a78 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py @@ -14,6 +14,8 @@ from paddle.fluid.optimizer import GradientMergeOptimizer as GM from .meta_optimizer_base import MetaOptimizerBase +__all__ = [] + class GradientMergeOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py index 9a4ffd2fd02d4..4194cf13d2bbc 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py @@ -19,6 +19,8 @@ from ..base.private_helper_function import wait_server_ready import logging +__all__ = [] + class GraphExecutionOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py index 64d54ae3bab03..6d2474d9352f8 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py @@ -16,6 +16,8 @@ from .meta_optimizer_base import MetaOptimizerBase import logging +__all__ = [] + class LambOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py index 32c6be505a546..e1bf3722c191d 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py @@ -15,6 +15,8 @@ from .meta_optimizer_base import MetaOptimizerBase import logging +__all__ = [] + class LarsOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py index 91030f0762934..3340672e0f925 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py @@ -19,6 +19,8 @@ from .meta_optimizer_base import MetaOptimizerBase from .common import OpRole, OP_ROLE_KEY, CollectiveHelper, is_update_op +__all__ = [] + class LocalSGDOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py index a12ca50442b1c..3bbaa055c5e59 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py +++ b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py @@ -14,6 +14,8 @@ from paddle.fluid.optimizer import Optimizer +__all__ = [] + class MetaOptimizerBase(Optimizer): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py index dfa765364f357..ba2a0e84c7ab6 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py @@ -15,6 +15,8 @@ from paddle.fluid import compiler from .parameter_server_optimizer import ParameterServerOptimizer +__all__ = [] + class ParameterServerGraphOptimizer(ParameterServerOptimizer): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py index f6d2af0b416d2..88180221ff4ff 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py @@ -20,6 +20,8 @@ import platform from ..base.private_helper_function import wait_server_ready +__all__ = [] + class ParameterServerOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py index 1aa51a6671c17..a0bf4cc5bc097 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py @@ -22,6 +22,8 @@ from .meta_optimizer_base import MetaOptimizerBase from .common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY, CollectiveHelper, is_loss_grad_op, is_backward_op, is_optimizer_op +__all__ = [] + class PipelineOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py index 3a784c306257b..d79675448c042 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py @@ -14,6 +14,8 @@ from paddle.fluid.optimizer import RecomputeOptimizer as RO from .meta_optimizer_base import MetaOptimizerBase +__all__ = [] + class RecomputeOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py index 40ba77815663f..8e63635372984 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py @@ -17,6 +17,8 @@ from paddle.fluid import core +__all__ = [] + class FP16Utils(object): def __init__(self): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py index d5a012b147a99..fd74f28b69e19 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py @@ -14,6 +14,8 @@ from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole +__all__ = [] + class GradientClipHelper(object): def __init__(self, mp_ring_id): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py index 76803818453c9..f6741b165ce07 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py @@ -15,6 +15,8 @@ from ..common import is_optimizer_op, OP_ROLE_KEY, OpRole from paddle.fluid import core, unique_name +__all__ = [] + class OffloadHelper(object): cpu_place_type = 0 diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py index 5a43367cf1ad1..dd4e16b576fcf 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +__all__ = [] + class ProgramDeps(object): def __init__(self, block, start_vars, end_vars): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py index 92e36e0ec1fff..0c33a78120cb8 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py @@ -16,6 +16,8 @@ from paddle.distributed.fleet.meta_optimizers.sharding.utils import * from paddle.distributed.fleet.meta_optimizers.sharding.fp16_helper import FP16Utils +__all__ = [] + class Shard(object): def __init__(self, ): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py index 2833e8c6dac4b..ab0c79bca554c 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py @@ -14,6 +14,8 @@ from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_VAR_KEY +__all__ = [] + class WeightDecayHelper(object): def __init__(self): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py index db6925ace5a64..82e54a89e104f 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py @@ -37,7 +37,7 @@ logger.addHandler(ch) from functools import reduce -__all__ = ["ShardingOptimizer"] +__all__ = [] class ShardingOptimizer(MetaOptimizerBase): diff --git a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py index 2ba0195156082..5fbec7da0b5ed 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py @@ -19,6 +19,8 @@ from .meta_optimizer_base import MetaOptimizerBase from .common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY, CollectiveHelper, is_update_op, is_loss_grad_op, is_backward_op, is_optimizer_op +__all__ = [] + class TensorParallelOptimizer(MetaOptimizerBase): def __init__(self, optimizer): diff --git a/python/paddle/distributed/fleet/meta_parallel/__init__.py b/python/paddle/distributed/fleet/meta_parallel/__init__.py index ed1add1f7baee..ed74d8e744e50 100644 --- a/python/paddle/distributed/fleet/meta_parallel/__init__.py +++ b/python/paddle/distributed/fleet/meta_parallel/__init__.py @@ -12,6 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .parallel_layers import * -from .model_parallel import ModelParallel -from .pipeline_parallel import PipelineParallel +from .parallel_layers import VocabParallelEmbedding # noqa: F401 +from .parallel_layers import ColumnParallelLinear # noqa: F401 +from .parallel_layers import RowParallelLinear # noqa: F401 +from .parallel_layers import LayerDesc # noqa: F401 +from .parallel_layers import PipelineLayer # noqa: F401 +from .parallel_layers import RNGStatesTracker # noqa: F401 +from .parallel_layers import model_parallel_random_seed # noqa: F401 +from .parallel_layers import get_rng_state_tracker # noqa: F401 +from .model_parallel import ModelParallel # noqa: F401 +from .pipeline_parallel import PipelineParallel # noqa: F401 + +__all__ = [] diff --git a/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py b/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py index cdf947895b777..69e41ab0edab2 100644 --- a/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py +++ b/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py @@ -14,6 +14,8 @@ from paddle.fluid.dygraph.layers import Layer +__all__ = [] + class MetaParallelBase(Layer): def __init__(self, layers, hcg, strategy): diff --git a/python/paddle/distributed/fleet/meta_parallel/model_parallel.py b/python/paddle/distributed/fleet/meta_parallel/model_parallel.py index ebf26498d9324..682d7152a42bd 100644 --- a/python/paddle/distributed/fleet/meta_parallel/model_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/model_parallel.py @@ -14,9 +14,13 @@ from paddle.fluid.dygraph.layers import Layer from .meta_parallel_base import MetaParallelBase -from ..utils.hybrid_parallel_util import * +from ..utils.hybrid_parallel_util import broadcast_dp_parameters +from ..utils.hybrid_parallel_util import broadcast_input_data +from ..utils.hybrid_parallel_util import broadcast_mp_parameters from ..utils.log_util import logger +__all__ = [] + class ModelParallel(MetaParallelBase): def __init__(self, layers, hcg, **kwargs): diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/__init__.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/__init__.py index c4ec61e84ffa5..6a33611403ace 100644 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/__init__.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/__init__.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .mp_layers import * -from .pp_layers import * -from .random import * +from .mp_layers import VocabParallelEmbedding # noqa: F401 +from .mp_layers import ColumnParallelLinear # noqa: F401 +from .mp_layers import RowParallelLinear # noqa: F401 +from .pp_layers import LayerDesc # noqa: F401 +from .pp_layers import PipelineLayer # noqa: F401 +from .random import RNGStatesTracker # noqa: F401 +from .random import model_parallel_random_seed # noqa: F401 +from .random import get_rng_state_tracker # noqa: F401 + +__all__ = [] diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py index b89e90128b112..af59b16e22aa8 100644 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py @@ -19,9 +19,7 @@ from paddle import framework from ...base import topology as tp -__all__ = [ - 'VocabParallelEmbedding', 'ColumnParallelLinear', 'RowParallelLinear' -] +__all__ = [] # Follow this paper to achieve the file: # Shoeybi M, Patwary M, Puri R, et al. Megatron-lm: Training multi-billion parameter diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py index a9704e38f3fa7..77be62ae6cf4b 100644 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py @@ -16,7 +16,7 @@ from paddle.fluid.dygraph.layers import Layer from ...utils.log_util import logger, layer_to_str -__all__ = ['LayerDesc', 'PipelineLayer'] +__all__ = [] class SegmentLayers(object): diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py index 56c741dbd3cad..41c9deabd1e11 100644 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py @@ -14,9 +14,8 @@ import paddle import contextlib -__all__ = [ - 'RNGStatesTracker', 'model_parallel_random_seed', 'get_rng_state_tracker' -] + +__all__ = [] MODEL_PARALLEL_RNG = 'model_parallel_rng' diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 8fb29a4485df0..79e5bc2ffeda0 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -25,9 +25,20 @@ from .pp_utils.utils import get_tensor_bytes, is_float_tensor from .pp_utils import utils from .parallel_layers.pp_layers import PipelineLayer -from ..utils.hybrid_parallel_util import * + +from ..utils.hybrid_parallel_util import broadcast_mp_parameters +from ..utils.hybrid_parallel_util import broadcast_dp_parameters +from ..utils.hybrid_parallel_util import fused_allreduce_gradients from ..utils.log_util import logger +__all__ = [] + +FLOAT_TYPES = [ + paddle.float16, + paddle.float32, + paddle.float64, +] + class PipelineParallel(MetaParallelBase): def __init__(self, layers, hcg, strategy): diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/__init__.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/__init__.py index d39e6760a3865..786eb20487a52 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/__init__.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/__init__.py @@ -12,4 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .utils import * +from .utils import get_tensor_bytes + +__all__ = [] diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py index 7b426e2c3f77d..e5c5709f98d95 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py @@ -16,10 +16,7 @@ import paddle from ...utils import hybrid_parallel_util as hp_util -__all__ = [ - 'get_tensor_bytes', - 'is_float_tensor', -] +__all__ = [] FLOAT_TYPES = [ paddle.float16, diff --git a/python/paddle/distributed/fleet/metrics/__init__.py b/python/paddle/distributed/fleet/metrics/__init__.py index bc30c063787d2..abcb90afb23c4 100644 --- a/python/paddle/distributed/fleet/metrics/__init__.py +++ b/python/paddle/distributed/fleet/metrics/__init__.py @@ -12,15 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .metric import * +from .metric import acc # noqa: F401 +from .metric import auc # noqa: F401 +from .metric import mae # noqa: F401 +from .metric import max # noqa: F401 +from .metric import min # noqa: F401 +from .metric import mse # noqa: F401 +from .metric import rmse # noqa: F401 +from .metric import sum # noqa: F401 -__all__ = [ - "sum", - "max", - "min", - "auc", - "mae", - "rmse", - "mse", - "acc", -] +__all__ = [] diff --git a/python/paddle/distributed/fleet/metrics/metric.py b/python/paddle/distributed/fleet/metrics/metric.py index 9ed0a0df4be01..d2050585df754 100644 --- a/python/paddle/distributed/fleet/metrics/metric.py +++ b/python/paddle/distributed/fleet/metrics/metric.py @@ -18,6 +18,8 @@ from paddle.static import Variable import paddle +__all__ = [] + def sum(input, scope=None, util=None): """ diff --git a/python/paddle/distributed/fleet/runtime/__init__.py b/python/paddle/distributed/fleet/runtime/__init__.py index 51d8c6ffebf1d..f5c30b2f3c5aa 100644 --- a/python/paddle/distributed/fleet/runtime/__init__.py +++ b/python/paddle/distributed/fleet/runtime/__init__.py @@ -15,3 +15,5 @@ from .collective_runtime import CollectiveRuntime from .parameter_server_runtime import ParameterServerRuntime from .the_one_ps import TheOnePSRuntime + +__all__ = [] diff --git a/python/paddle/distributed/fleet/runtime/collective_runtime.py b/python/paddle/distributed/fleet/runtime/collective_runtime.py index c56cf4c7aa2ed..a23b15f1fca1b 100644 --- a/python/paddle/distributed/fleet/runtime/collective_runtime.py +++ b/python/paddle/distributed/fleet/runtime/collective_runtime.py @@ -15,6 +15,8 @@ from .runtime_base import RuntimeBase import logging +__all__ = [] + class CollectiveRuntime(RuntimeBase): def __init__(self): diff --git a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py index 782ba87e07925..0767158d23f00 100644 --- a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py +++ b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py @@ -26,6 +26,8 @@ from .runtime_base import RuntimeBase from ..base.private_helper_function import wait_server_ready +__all__ = [] + class ParameterServerRuntime(RuntimeBase): def __init__(self): diff --git a/python/paddle/distributed/fleet/runtime/the_one_ps.py b/python/paddle/distributed/fleet/runtime/the_one_ps.py index df07a7a6e7783..5dd0419178642 100644 --- a/python/paddle/distributed/fleet/runtime/the_one_ps.py +++ b/python/paddle/distributed/fleet/runtime/the_one_ps.py @@ -25,6 +25,8 @@ from .runtime_base import RuntimeBase from ..base.private_helper_function import wait_server_ready +__all__ = [] + def conv_indent(indent): return "".join([" "] * indent) diff --git a/python/paddle/distributed/fleet/utils/__init__.py b/python/paddle/distributed/fleet/utils/__init__.py index 0a47750ead7ec..1bf90a22e375c 100644 --- a/python/paddle/distributed/fleet/utils/__init__.py +++ b/python/paddle/distributed/fleet/utils/__init__.py @@ -12,6 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .fs import LocalFS, HDFSClient -from .ps_util import DistributedInfer -from .recompute import recompute +from .fs import LocalFS # noqa: F401 +from .fs import HDFSClient # noqa: F401 +from .ps_util import DistributedInfer # noqa: F401 +from .recompute import recompute # noqa: F401 + +from . import log_util # noqa: F401 +from . import hybrid_parallel_util # noqa: F401 + +__all__ = [ #noqa + "LocalFS", "recompute", "DistributedInfer", "HDFSClient" +] diff --git a/python/paddle/distributed/fleet/utils/fs.py b/python/paddle/distributed/fleet/utils/fs.py index 7e62e551fe8d5..087942e70a226 100644 --- a/python/paddle/distributed/fleet/utils/fs.py +++ b/python/paddle/distributed/fleet/utils/fs.py @@ -31,7 +31,7 @@ import shutil -__all__ = ['LocalFS', 'HDFSClient'] +__all__ = [] class ExecuteError(Exception): diff --git a/python/paddle/distributed/fleet/utils/http_server.py b/python/paddle/distributed/fleet/utils/http_server.py index 92295cc74ae4d..a9d0687461b99 100644 --- a/python/paddle/distributed/fleet/utils/http_server.py +++ b/python/paddle/distributed/fleet/utils/http_server.py @@ -28,6 +28,8 @@ import threading import socket +__all__ = [] + def get_logger(name, level, fmt): logger = logging.getLogger(name) diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py index de2d3f45ba033..5521bd5b95283 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py @@ -23,6 +23,8 @@ from collections import OrderedDict from .log_util import logger +__all__ = [] + def _apply_collective_grads(parameters, comm_group): grad_var_set = set() diff --git a/python/paddle/distributed/fleet/utils/log_util.py b/python/paddle/distributed/fleet/utils/log_util.py index 12c0bf699c1e6..77eb641e0c6fe 100644 --- a/python/paddle/distributed/fleet/utils/log_util.py +++ b/python/paddle/distributed/fleet/utils/log_util.py @@ -15,6 +15,8 @@ import logging import sys +__all__ = [] + class LoggerFactory: @staticmethod diff --git a/python/paddle/distributed/fleet/utils/ps_util.py b/python/paddle/distributed/fleet/utils/ps_util.py index 7bf7bec43de00..8bf69a41a7cc8 100644 --- a/python/paddle/distributed/fleet/utils/ps_util.py +++ b/python/paddle/distributed/fleet/utils/ps_util.py @@ -18,6 +18,8 @@ import paddle import warnings +__all__ = [] + class DistributedInfer: """ diff --git a/python/paddle/distributed/fleet/utils/recompute.py b/python/paddle/distributed/fleet/utils/recompute.py index d61c3cfd1e578..e58c8aa1625dd 100644 --- a/python/paddle/distributed/fleet/utils/recompute.py +++ b/python/paddle/distributed/fleet/utils/recompute.py @@ -26,6 +26,8 @@ ch.setFormatter(formatter) logger.addHandler(ch) +__all__ = [] + def detach_variable(inputs): out = [] diff --git a/python/paddle/distributed/launch.py b/python/paddle/distributed/launch.py index df3a3407bf5cf..e02a439025b77 100644 --- a/python/paddle/distributed/launch.py +++ b/python/paddle/distributed/launch.py @@ -14,3 +14,5 @@ from paddle.distributed.fleet import launch launch.launch() + +__all__ = [] diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py index 582c0be713f4e..bc042e722947a 100644 --- a/python/paddle/distributed/parallel.py +++ b/python/paddle/distributed/parallel.py @@ -15,7 +15,8 @@ import os import six import warnings -from multiprocessing import Process, Manager +from multiprocessing import Process # noqa: F401 +from multiprocessing import Manager # noqa: F401 import time import sys @@ -26,9 +27,11 @@ from paddle.fluid.framework import _set_expected_place from paddle.fluid.dygraph import parallel_helper from paddle.fluid.dygraph.parallel import ParallelEnv -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import wait_server_ready # noqa: F401 -__all__ = ["init_parallel_env"] +__all__ = [ #noqa + "init_parallel_env" +] ParallelStrategy = core.ParallelStrategy diff --git a/python/paddle/distributed/spawn.py b/python/paddle/distributed/spawn.py index 782fcb28e991c..c46672dca09e9 100644 --- a/python/paddle/distributed/spawn.py +++ b/python/paddle/distributed/spawn.py @@ -21,7 +21,9 @@ import sys import warnings -from paddle.distributed.utils import _print_arguments, _prepare_trainer_env, get_host_name_ip +from paddle.distributed.utils import _print_arguments +from paddle.distributed.utils import _prepare_trainer_env +from paddle.distributed.utils import get_host_name_ip from paddle.distributed.cloud_utils import get_cluster_and_pod from paddle.distributed.fleet.cloud_utils import use_paddlecloud from paddle.device import get_device @@ -30,6 +32,8 @@ from paddle.fluid import core from paddle.fluid.framework import _cpu_num, set_flags +__all__ = [] + class ParallelEnvArgs(object): def __init__(self): diff --git a/python/paddle/distributed/utils.py b/python/paddle/distributed/utils.py index f40a7b31b83e6..e84025c2eb6d2 100644 --- a/python/paddle/distributed/utils.py +++ b/python/paddle/distributed/utils.py @@ -26,6 +26,24 @@ import socket from paddle.fluid import core +__all__ = [ #noqa + 'get_host_name_ip', + 'Trainer', + 'get_cluster', + 'start_local_trainers', + 'watch_local_trainers', + 'find_free_ports', + 'JobServer', + 'Cluster', + 'Pod', + 'Hdfs', + 'add_arguments', + 'terminate_local_procs', + 'TrainerProc', + 'get_logger', + 'pull_worker_log' +] + logger = logging.getLogger("root") logger.propagate = False diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index b5a6a5ca07384..7cf3f94872de1 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -206,7 +206,7 @@ def weight_norm(*args): 'Dropout3D', 'Bilinear', 'AlphaDropout', - 'Unfold' + 'Unfold', 'RNNCellBase', 'SimpleRNNCell', 'LSTMCell', From 4026e2271464fbf4f69885e5252921ceb8017e96 Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Tue, 25 May 2021 12:00:51 +0800 Subject: [PATCH 067/156] [HybridParallel]Fix precision problem of model parallel (#32897) (#33087) * fix precision of mp * fix bug of seed * fix dp * print group --- .../framework/distributed_strategy.proto | 1 + python/paddle/distributed/collective.py | 7 + .../fleet/base/distributed_strategy.py | 5 +- .../distributed/fleet/base/fleet_base.py | 15 +- .../paddle/distributed/fleet/base/topology.py | 6 +- .../hybrid_parallel_gradscaler.py | 2 +- .../hybrid_parallel_optimizer.py | 4 +- .../fleet/meta_parallel/__init__.py | 2 +- .../parallel_layers/mp_layers.py | 135 +++++++++++++----- .../meta_parallel/parallel_layers/random.py | 13 +- .../{model_parallel.py => tensor_parallel.py} | 6 +- .../fleet/utils/hybrid_parallel_util.py | 10 +- .../unittests/hybrid_parallel_mp_layers.py | 2 +- .../paddle/fluid/tests/unittests/new_group.py | 1 + 14 files changed, 151 insertions(+), 58 deletions(-) rename python/paddle/distributed/fleet/meta_parallel/{model_parallel.py => tensor_parallel.py} (89%) diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 99a6eb6b67472..38831192c8c2b 100644 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -141,6 +141,7 @@ message PipelineConfig { message TensorParallelConfig { optional int32 tensor_parallel_degree = 1 [ default = 1 ]; + optional int32 tensor_init_seed = 2 [ default = -1 ]; } message DistributedStrategy { diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index 85b8cafd6c315..55f86959c59f2 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -98,6 +98,13 @@ def get_group_rank(self, rank): else: return -1 + def __repr__(self): + debug_str = "rank: {}, nranks: {}, id: {}, ranks: ".format( + self.rank, self.nranks, self.id) + debug_str += ", ".join(map(str, self.ranks)) + debug_str += ". " + return debug_str + _global_env = None diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index 25e571dba0c80..640bc00cb6c57 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -923,6 +923,8 @@ def tensor_parallel_configs(self): **Notes**: **Detailed arguments for tensor_parallel_configs** **tensor_parallel_degree**: degree of tensor parallel + **tensor_init_seed**: parameter initialization random seed + Examples: @@ -931,7 +933,8 @@ def tensor_parallel_configs(self): import paddle.distributed.fleet as fleet strategy = fleet.DistributedStrategy() strategy.tensor_parallel = True - strategy.tensor_parallel_configs = {"tensor_parallel_degree": 4} + strategy.tensor_parallel_configs = {"tensor_parallel_degree": 4, + "tensor_init_seed": 123} """ return get_msg_dict(self.strategy.tensor_parallel_configs) diff --git a/python/paddle/distributed/fleet/base/fleet_base.py b/python/paddle/distributed/fleet/base/fleet_base.py index a7564a23a7cfb..edc4a22dc37e9 100644 --- a/python/paddle/distributed/fleet/base/fleet_base.py +++ b/python/paddle/distributed/fleet/base/fleet_base.py @@ -17,6 +17,7 @@ import warnings import paddle import os +import numpy as np from paddle.fluid.framework import dygraph_only from paddle.fluid import compiler from .role_maker import UserDefinedRoleMaker, PaddleCloudRoleMaker, RoleMakerBase @@ -28,7 +29,7 @@ from paddle.fluid.dygraph import parallel_helper from . import topology as tp from .topology import ParallelMode -from ..meta_parallel import ModelParallel +from ..meta_parallel import TensorParallel, model_parallel_random_seed from ..meta_parallel import PipelineParallel from ..meta_optimizers import HybridParallelOptimizer from ..meta_optimizers import HybridParallelGradScaler @@ -279,6 +280,14 @@ def _init_hybrid_parallel_env(self): self._hcg = tp.HybridCommunicateGroup(self._topology) + if self.mp_degree > 1: + tensor_parallel_configs = self._user_defined_strategy.tensor_parallel_configs + tensor_init_seed = tensor_parallel_configs["tensor_init_seed"] + if tensor_init_seed == -1: + model_parallel_random_seed() + else: + model_parallel_random_seed(tensor_init_seed) + def get_hybrid_communicate_group(self): assert self._hcg is not None return self._hcg @@ -780,8 +789,8 @@ def forward(self, x): last_comm_group_size_MB, find_unused_parameters=self._user_defined_strategy. find_unused_parameters) - elif self._hcg.get_parallel_mode() == ParallelMode.MODEL_PARALLEL: - distributed_model = ModelParallel( + elif self._hcg.get_parallel_mode() == ParallelMode.TENSOR_PARALLEL: + distributed_model = TensorParallel( model, self._hcg, strategy=self._user_defined_strategy) elif self._hcg.get_parallel_mode() == ParallelMode.PIPELINE_PARALLEL: distributed_model = PipelineParallel( diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py index 470a4d83aac3f..04525977192be 100644 --- a/python/paddle/distributed/fleet/base/topology.py +++ b/python/paddle/distributed/fleet/base/topology.py @@ -28,7 +28,7 @@ class ParallelMode(object): DATA_PARALLEL = 0 - MODEL_PARALLEL = 1 + TENSOR_PARALLEL = 1 PIPELINE_PARALLEL = 2 @@ -155,12 +155,12 @@ def __init__(self, topology): _HYBRID_PARALLEL_GROUP = self def get_parallel_mode(self): - # there are three modes : DataParallel / ModelParallel / PipelineParallel + # there are three modes : DataParallel / TensorParallel / PipelineParallel if self._mp_degree == 1 and self._pp_degree == 1: return ParallelMode.DATA_PARALLEL elif self._mp_degree > 1 and self._pp_degree == 1: # initialize the seed - return ParallelMode.MODEL_PARALLEL + return ParallelMode.TENSOR_PARALLEL elif self._pp_degree > 1: return ParallelMode.PIPELINE_PARALLEL diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py index d0e8034f5cae1..c0f671e7e446b 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py @@ -31,7 +31,7 @@ def __init__(self, scaler, hcg): self._scaler = scaler self._hcg = hcg self._is_mp = ( - self._hcg.get_parallel_mode() == ParallelMode.MODEL_PARALLEL) + self._hcg.get_parallel_mode() == ParallelMode.TENSOR_PARALLEL) def scale(self, var): return self._scaler.scale(var) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py index b7ac298d2223e..00ac019c0d188 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py @@ -90,12 +90,12 @@ def __init__(self, optimizer, hcg, strategy): self._strategy = strategy self._hcg = hcg self._is_mp = ( - self._hcg.get_parallel_mode() == ParallelMode.MODEL_PARALLEL) + self._hcg.get_parallel_mode() == ParallelMode.TENSOR_PARALLEL) self._need_dp = (self._hcg.get_data_parallel_world_size() > 1) if isinstance(self._inner_opt._grad_clip, ClipGradByGlobalNorm) and self._is_mp: - logger.warning("using ClipGradByGlobalNorm in ModelParallel, the origin " \ + logger.warning("using ClipGradByGlobalNorm in TensorParallel, the origin " \ "optmizer'grad clip will be changed.") self._inner_opt._grad_clip = HybridParallelClipGrad( self._inner_opt._grad_clip, hcg) diff --git a/python/paddle/distributed/fleet/meta_parallel/__init__.py b/python/paddle/distributed/fleet/meta_parallel/__init__.py index ed74d8e744e50..894771a3d5005 100644 --- a/python/paddle/distributed/fleet/meta_parallel/__init__.py +++ b/python/paddle/distributed/fleet/meta_parallel/__init__.py @@ -20,7 +20,7 @@ from .parallel_layers import RNGStatesTracker # noqa: F401 from .parallel_layers import model_parallel_random_seed # noqa: F401 from .parallel_layers import get_rng_state_tracker # noqa: F401 -from .model_parallel import ModelParallel # noqa: F401 +from .tensor_parallel import TensorParallel # noqa: F401 from .pipeline_parallel import PipelineParallel # noqa: F401 __all__ = [] diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py index af59b16e22aa8..730a7430133e0 100644 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/mp_layers.py @@ -41,6 +41,7 @@ def __init__(self, self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() self.origin_num_embeddings = num_embeddings + self.is_mp = (self.world_size > 1) per_part_size = ( num_embeddings + self.world_size - 1) // self.world_size @@ -50,16 +51,36 @@ def __init__(self, per_part_size += 1 # make the last row as the padding index self.per_part_size = per_part_size - self.embedding = paddle.nn.Embedding( - per_part_size, - embedding_dim, - padding_idx=per_part_size - 1, - sparse=False, - weight_attr=weight_attr, - name=name) - self.embedding.weight.is_distributed = True + self._dtype = self._helper.get_default_dtype() + self._size = [per_part_size, embedding_dim] + self._weight_attr = weight_attr + self._name = name + + if self.is_mp: + with get_rng_state_tracker().rng_state(): + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False) + self.weight[per_part_size - 1] = 0.0 + self.weight.is_distributed = True + else: + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=[num_embeddings, embedding_dim], + dtype=self._dtype, + is_bias=False) def forward(self, x): + if not self.is_mp: + return F.embedding( + x, + weight=self.weight, + padding_idx=None, + sparse=False, + name=self._name) + origin_input_shape = x.shape if len(origin_input_shape) == 2: x = paddle.unsqueeze(x, axis=-1) @@ -72,13 +93,18 @@ def forward(self, x): if len(origin_input_shape) == 2: x_shard = paddle.squeeze(x_shard, axis=-1) - emb_out = self.embedding(x_shard) - if self.world_size > 1: - emb_out = paddle.distributed.collective._mp_allreduce( - emb_out, - group=self.model_parallel_group, - use_calc_stream=True, - use_model_parallel=True) + emb_out = F.embedding( + x_shard, + weight=self.weight, + padding_idx=self.per_part_size - 1, + sparse=False, + name=self._name) + + emb_out = paddle.distributed.collective._mp_allreduce( + emb_out, + group=self.model_parallel_group, + use_calc_stream=True, + use_model_parallel=True) return emb_out @@ -96,8 +122,9 @@ def __init__(self, ) self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( ) + self._name = name + self.is_mp = (self.world_size > 1) - self.name = name self.gather_output = gather_output assert out_features % self.world_size == 0, ( "Number of column of the weight for linear ({}) must be" @@ -108,10 +135,20 @@ def __init__(self, self._weight_attr = weight_attr self._dtype = self._helper.get_default_dtype() - self.weight = self.create_parameter( - shape=[in_features, self.output_size_per_partition], - attr=self._weight_attr, - dtype=self._dtype) + if self.is_mp: + with get_rng_state_tracker().rng_state(): + self.weight = self.create_parameter( + shape=[in_features, self.output_size_per_partition], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False) + else: + self.weight = self.create_parameter( + shape=[in_features, self.output_size_per_partition], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False) + self.weight.is_distributed = True if has_bias: @@ -119,18 +156,24 @@ def __init__(self, self.bias = self.create_parameter( shape=[self.output_size_per_partition], attr=paddle.nn.initializer.Constant(value=0.0), - dtype=self._dtype) + dtype=self._dtype, + is_bias=True) self.bias.is_distributed = True else: self.bias = None def forward(self, x): # use inner api to process identity - input_parallel = paddle.distributed.collective._c_identity( - x, group=self.model_parallel_group) + if self.is_mp: + input_parallel = paddle.distributed.collective._c_identity( + x, group=self.model_parallel_group) + else: + input_parallel = x + output_parallel = F.linear( - input_parallel, self.weight, self.bias, name=self.name) - if self.gather_output: + input_parallel, self.weight, self.bias, name=self._name) + + if self.gather_output and self.is_mp: output = paddle.distributed.collective._c_concat( output_parallel, nranks=self.world_size, @@ -155,7 +198,7 @@ def __init__(self, self.input_is_parallel = input_is_parallel self._weight_attr = weight_attr self._dtype = self._helper.get_default_dtype() - self.name = name + self._name = name self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( ) @@ -163,6 +206,7 @@ def __init__(self, ) self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() + self.is_mp = (self.world_size > 1) assert in_features % self.world_size == 0, ( "Number of row of the weight for linear ({}) must be" " divisible by model parallel size ({})".format(in_features, @@ -170,22 +214,33 @@ def __init__(self, self.input_size_per_partition = in_features // self.world_size - self.weight = self.create_parameter( - shape=[self.input_size_per_partition, self.out_features], - attr=self._weight_attr, - dtype=self._dtype) + if self.is_mp: + with get_rng_state_tracker().rng_state(): + self.weight = self.create_parameter( + shape=[self.input_size_per_partition, self.out_features], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False) + else: + self.weight = self.create_parameter( + shape=[self.input_size_per_partition, self.out_features], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False) + self.weight.is_distributed = True if has_bias: self.bias = self.create_parameter( shape=[self.out_features], attr=paddle.nn.initializer.Constant(value=0.0), - dtype=self._dtype) + dtype=self._dtype, + is_bias=True) else: self.bias = None def forward(self, x): - if self.input_is_parallel: + if self.input_is_parallel or (not self.is_mp): input_parallel = x else: # split last dim @@ -195,12 +250,16 @@ def forward(self, x): nranks=self.world_size, group=self.model_parallel_group) - output_parallel = F.linear(input_parallel, self.weight, name=self.name) - output_ = paddle.distributed.collective._mp_allreduce( - output_parallel, - group=self.model_parallel_group, - use_calc_stream=True, - use_model_parallel=True) + output_parallel = F.linear(input_parallel, self.weight, name=self._name) + + if self.is_mp: + output_ = paddle.distributed.collective._mp_allreduce( + output_parallel, + group=self.model_parallel_group, + use_calc_stream=True, + use_model_parallel=True) + else: + output_ = output_parallel output = output_ + self.bias if self.bias is not None else output_ return output diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py index 41c9deabd1e11..70daa3b25365e 100644 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/random.py @@ -14,6 +14,7 @@ import paddle import contextlib +import numpy as np __all__ = [] @@ -65,14 +66,18 @@ def get_rng_state_tracker(): return RNG_STATE_TRACKER -def model_parallel_random_seed(seed=2048): +def model_parallel_random_seed(seed=None): import paddle.distributed.fleet as fleet hcg = fleet.get_hybrid_communicate_group() rank = hcg.get_model_parallel_rank() - local_seed = seed + 1024 + rank - global_seed = seed + if seed: + global_seed = seed + local_seed = seed * 1024 + rank * 100 + else: + global_seed = np.random.randint(0, 655350) + local_seed = np.random.randint(rank * 10000, (rank + 1) * 10000 - 1) RNG_STATE_TRACKER.reset() - paddle.seed(global_seed) RNG_STATE_TRACKER.add(MODEL_PARALLEL_RNG, local_seed) + paddle.seed(global_seed) diff --git a/python/paddle/distributed/fleet/meta_parallel/model_parallel.py b/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py similarity index 89% rename from python/paddle/distributed/fleet/meta_parallel/model_parallel.py rename to python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py index 682d7152a42bd..1dbf668d6e13a 100644 --- a/python/paddle/distributed/fleet/meta_parallel/model_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py @@ -22,15 +22,15 @@ __all__ = [] -class ModelParallel(MetaParallelBase): +class TensorParallel(MetaParallelBase): def __init__(self, layers, hcg, **kwargs): - super(ModelParallel, self).__init__(layers, hcg, **kwargs) + super(TensorParallel, self).__init__(layers, hcg, **kwargs) def _prepare_for_model(self): logger.info("start broadcast mp parameters") broadcast_mp_parameters(self._layers, self._hcg) - logger.info("start broadcast mp parameters") + logger.info("start broadcast dp parameters") broadcast_dp_parameters(self._layers, self._hcg) logger.info("mp's parameters is ready") diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py index 5521bd5b95283..ddbd6111b4609 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py @@ -44,7 +44,15 @@ def _apply_collective_grads(parameters, comm_group): for coalesced_grad, _, _ in coalesced_grads_and_vars: # need to div nranks - coalesced_grad = coalesced_grad / comm_group.nranks + div_factor = paddle.to_tensor( + comm_group.nranks, dtype=coalesced_grad.dtype) + paddle.fluid.framework._dygraph_tracer().trace_op( + type="elementwise_div", + inputs={'X': coalesced_grad, + 'Y': div_factor}, + outputs={'Out': coalesced_grad}, + attrs={'axis': -1}) + paddle.distributed.all_reduce(coalesced_grad, group=comm_group) _split_tensors(coalesced_grads_and_vars) diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_layers.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_layers.py index dfbef998a2f07..349d5f82dbf54 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_layers.py @@ -231,7 +231,7 @@ def test_parallel_embedding(self): # model_b check_group = dist.new_group(list(range(self.model_parallel_size))) integral_w = [] - partial_w = model_a.embedding.embedding.weight.clone().detach() + partial_w = model_a.embedding.weight.clone().detach() paddle.distributed.all_gather(integral_w, partial_w, group=check_group) result_w = [] for idx in range(len(integral_w)): diff --git a/python/paddle/fluid/tests/unittests/new_group.py b/python/paddle/fluid/tests/unittests/new_group.py index fb7beeee1df2e..c9c4acc3220c7 100644 --- a/python/paddle/fluid/tests/unittests/new_group.py +++ b/python/paddle/fluid/tests/unittests/new_group.py @@ -27,6 +27,7 @@ def __init__(self): def test_all(self): gp = paddle.distributed.new_group([0, 1]) + print("gp info:", gp) print("test new group api ok") tmp = np.array([0, 0, 0]) From 8fe6d559939b83ca856bbc462fed22ebd5f1507b Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Wed, 26 May 2021 14:14:30 +0800 Subject: [PATCH 068/156] [Cherry-pick][Dy2Stat]Support convert sublayers in Sequential Container (#32978) (#33065) * Support convert sublayers in Sequential Container * remove paddle.jit.set_code_level --- .../dygraph_to_static/convert_call_func.py | 8 ++ .../dygraph_to_static/test_container.py | 91 +++++++++++++++++++ .../unittests/dygraph_to_static/test_list.py | 3 - 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py index 7604be2d838eb..a621f68c6545a 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py @@ -26,6 +26,7 @@ import numpy import six +from paddle.fluid.dygraph.container import Sequential from paddle.fluid.dygraph.dygraph_to_static.convert_operators import convert_len from paddle.fluid.dygraph.dygraph_to_static.logging_utils import TranslatorLogger from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction @@ -40,6 +41,9 @@ BUILTIN_LIKELY_MODULES = [ collections, pdb, copy, inspect, re, six, numpy, logging ] +# The api(s) should be considered as plain function and convert +# them into static layer code. +PADDLE_NEED_CONVERT_APIS = [Sequential] translator_logger = TranslatorLogger() @@ -92,6 +96,10 @@ def is_unsupported(func): format(func)) return True + # NOTE: should be placed before `is_paddle_func` + if type(func) in PADDLE_NEED_CONVERT_APIS: + return False + if is_paddle_func(func): translator_logger.log( 2, diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py new file mode 100644 index 0000000000000..647c9e9672cf0 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import unittest +import numpy as np + + +class BufferLayers(paddle.nn.Layer): + def __init__(self, out_channel): + super(BufferLayers, self).__init__() + self.out_channel = out_channel + + def forward(self, x): + mean = paddle.mean(x) + if mean < 0.: + x = x * self._mask() + + out = x - mean + return out + + def _mask(self): + return paddle.to_tensor(np.zeros([self.out_channel], 'float32')) + + +class SequentialNet(paddle.nn.Layer): + def __init__(self, sub_layer, in_channel, out_channel): + super(SequentialNet, self).__init__() + self.layer = paddle.nn.Sequential( + ('l1', paddle.nn.Linear(in_channel, in_channel)), + ('l2', paddle.nn.Linear(in_channel, out_channel)), + ('l3', sub_layer(out_channel))) + + def forward(self, x): + out = self.layer(x) + return out + + +class TestSequential(unittest.TestCase): + def setUp(self): + paddle.set_device('cpu') + self.seed = 2021 + + def _init_seed(self): + paddle.seed(self.seed) + np.random.seed(self.seed) + + def _run(self, to_static): + self._init_seed() + net = SequentialNet(BufferLayers, 10, 3) + if to_static: + net = paddle.jit.to_static(net) + x = paddle.rand([16, 10], 'float32') + out = net(x) + if to_static: + load_out = self._test_load(net, x) + self.assertTrue( + np.allclose(load_out, out), + msg='load_out is {}\st_out is {}'.format(load_out, out)) + + return out + + def test_train(self): + paddle.jit.set_code_level(100) + dy_out = self._run(to_static=False) + st_out = self._run(to_static=True) + self.assertTrue( + np.allclose(dy_out, st_out), + msg='dygraph_res is {}\nstatic_res is {}'.format(dy_out, st_out)) + + def _test_load(self, net, x): + model_path = './sequential_net' + paddle.jit.save(net, model_path) + load_net = paddle.jit.load(model_path) + out = load_net(x) + return out + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py index e630c2b9c6feb..8da4e200cfc36 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py @@ -62,9 +62,6 @@ def test_list_append_in_for_loop(x, iter_num): return a[0] -paddle.jit.set_code_level(100) - - def test_list_append_in_for_subscript(x): x = fluid.dygraph.to_variable(x) iter_num = paddle.shape(x)[0] From d7d3090fb2d5a43d683baa663c63a4079cf71f77 Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Wed, 26 May 2021 17:10:24 +0800 Subject: [PATCH 069/156] [Cherry-Pick][HybridParallel]Fix pipeline in dygraph (#33097) * [HybridParallel]Fix pipeline in dygraph (#33007) * fix pipeline * fix mp pp dp * fix utest of hybrid parallel * add utest for tuple * fix utest (#33108) --- .../paddle/distributed/fleet/base/topology.py | 5 + .../hybrid_parallel_optimizer.py | 12 +- .../fleet/meta_parallel/pipeline_parallel.py | 325 ++++++++++-------- .../fleet/meta_parallel/pp_utils/utils.py | 120 ++----- .../fluid/tests/unittests/CMakeLists.txt | 11 +- .../unittests/hybrid_parallel_mp_model.py | 40 +-- .../unittests/hybrid_parallel_pp_alexnet.py | 120 +++++++ .../unittests/hybrid_parallel_pp_embedding.py | 208 +++++++++++ .../unittests/hybrid_parallel_pp_layer.py | 34 +- .../unittests/hybrid_parallel_pp_model.py | 93 ----- .../test_parallel_dygraph_dataparallel.py | 54 ++- ...est_parallel_dygraph_pipeline_parallel.py} | 3 + ... test_parallel_dygraph_tensor_parallel.py} | 0 .../tests/unittests/test_pipeline_parallel.py | 2 +- 14 files changed, 649 insertions(+), 378 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py create mode 100644 python/paddle/fluid/tests/unittests/hybrid_parallel_pp_embedding.py delete mode 100644 python/paddle/fluid/tests/unittests/hybrid_parallel_pp_model.py rename python/paddle/fluid/tests/unittests/{test_parallel_dygraph_pipeline_layer.py => test_parallel_dygraph_pipeline_parallel.py} (89%) rename python/paddle/fluid/tests/unittests/{test_parallel_dygraph_hybrid_parallel.py => test_parallel_dygraph_tensor_parallel.py} (100%) diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py index 04525977192be..04d8417fdcbf3 100644 --- a/python/paddle/distributed/fleet/base/topology.py +++ b/python/paddle/distributed/fleet/base/topology.py @@ -253,3 +253,8 @@ def get_pipe_parallel_group(self): # check parallel group def get_check_parallel_group(self): return self._check_comm_group + + def get_rank_from_stage(self, stage_id): + coord = self._topo.get_coord(self.global_rank) + tf = coord._replace(pipe=stage_id)._asdict() + return self._topo.get_rank(**tf) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py index 00ac019c0d188..c2d79a62c7663 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py @@ -89,12 +89,14 @@ def __init__(self, optimizer, hcg, strategy): self._inner_opt = optimizer self._strategy = strategy self._hcg = hcg - self._is_mp = ( - self._hcg.get_parallel_mode() == ParallelMode.TENSOR_PARALLEL) + + self._use_dp_mode = ( + self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL) + self._need_dp = (self._hcg.get_data_parallel_world_size() > 1) if isinstance(self._inner_opt._grad_clip, - ClipGradByGlobalNorm) and self._is_mp: + ClipGradByGlobalNorm) and not self._use_dp_mode: logger.warning("using ClipGradByGlobalNorm in TensorParallel, the origin " \ "optmizer'grad clip will be changed.") self._inner_opt._grad_clip = HybridParallelClipGrad( @@ -103,7 +105,7 @@ def __init__(self, optimizer, hcg, strategy): @imperative_base.no_grad @framework.dygraph_only def step(self): - if self._is_mp and self._need_dp: + if not self._use_dp_mode and self._need_dp: fused_allreduce_gradients( list(self._inner_opt._parameter_list), self._hcg) self._inner_opt.step() @@ -119,7 +121,7 @@ def minimize(self, parameter_list = parameters if parameters \ else self._parameter_list - if self._is_mp and self._need_dp: + if not self._use_dp_mode and self._need_dp: fused_allreduce_gradients(list(parameter_list), self._hcg) return self._inner_opt.minimize(loss, startup_program, parameters, diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 79e5bc2ffeda0..54324b389336d 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -11,39 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -import time -import copy -import os - from types import MethodType -from numpy import prod - import paddle import paddle.fluid as fluid from .meta_parallel_base import MetaParallelBase -from .pp_utils.utils import get_tensor_bytes, is_float_tensor +from .pp_utils.utils import is_float_tensor, get_tensor_dtype, paddle_2_number, number_2_dtype from .pp_utils import utils from .parallel_layers.pp_layers import PipelineLayer from ..utils.hybrid_parallel_util import broadcast_mp_parameters from ..utils.hybrid_parallel_util import broadcast_dp_parameters -from ..utils.hybrid_parallel_util import fused_allreduce_gradients from ..utils.log_util import logger +from ..meta_optimizers.dygraph_optimizer import HybridParallelOptimizer __all__ = [] -FLOAT_TYPES = [ - paddle.float16, - paddle.float32, - paddle.float64, -] - class PipelineParallel(MetaParallelBase): def __init__(self, layers, hcg, strategy): + if not isinstance(layers, PipelineLayer): + raise TypeError( + "The Layer should be a derived class of PipelineLayer.") super(PipelineParallel, self).__init__(layers, hcg, strategy) - self.use_pipe_parallel = self._hcg.get_pipe_parallel_world_size() > 1 self.use_data_parallel = self._hcg.get_data_parallel_world_size() > 1 self.use_model_parallel = self._hcg.get_model_parallel_world_size() > 1 @@ -63,8 +53,6 @@ def __init__(self, layers, hcg, strategy): self.current_loss = paddle.to_tensor(0.0) self.total_loss = None - self.use_amp = self._strategy.amp - self.init_loss_scaling = self._strategy.amp_configs['init_loss_scaling'] self.micro_batch_size = self._strategy.pipeline_configs[ 'micro_batch_size'] self.accumulate_steps = self._strategy.pipeline_configs[ @@ -75,6 +63,11 @@ def __init__(self, layers, hcg, strategy): self.prev_stage_id = self.stage_id - 1 self.next_stage_id = self.stage_id + 1 self.pp_group = self._hcg.get_pipe_parallel_group() + + self.is_first_stage = self.stage_id == 0 + self.is_last_stage = (self.stage_id == (self.num_stages - 1)) + self.global_rank = self._hcg.get_global_rank() + logger.info("Pipeline Info -- num_stages: {}, stage_id: {}".format( self.num_stages, self.stage_id)) @@ -83,51 +76,72 @@ def __init__(self, layers, hcg, strategy): broadcast_mp_parameters(self._layers, self._hcg) if self.use_data_parallel: - logger.info("start broadcast mp parameters") + logger.info("start broadcast dp parameters") broadcast_dp_parameters(self._layers, self._hcg) - def _allocate_caches(self, num_caches): + def _init_caches(self, num_caches): if self.num_caches >= num_caches: return - - num = num_caches - self.num_caches - self.num_caches = num_caches + self.num_caches = num_caches - self.num_caches for key in self.caches: - self.caches[key].extend([None] * num) + self.caches[key].extend([None] * self.num_caches) + + def _reduce_final_loss(self): + if self.is_last_stage: + assert self.total_loss is not None, "train_batch() in last stage should obtain vaild loss" + loss = self.total_loss.clone() / self.accumulate_steps + paddle.distributed.broadcast( + loss, + src=self.global_rank, + use_calc_stream=True, + group=self.pp_group) + else: + loss = paddle.to_tensor(0.0) + paddle.distributed.broadcast( + loss, + src=self._hcg.get_rank_from_stage(self.num_stages - 1), + use_calc_stream=True, + group=self.pp_group) + return loss - def train_batch(self, data, optimizer): + def train_batch(self, data, optimizer, lr_scheduler=None): + assert isinstance(optimizer, HybridParallelOptimizer), ( + 'optimizer should be HybridParallelOptimizer subclass.') self.optimizer = optimizer + self.lr_scheduler = lr_scheduler assert fluid.framework._dygraph_tracer()._has_grad, ( 'Please enable the generation of gradients.') - if self.stage_id == 0 or self.stage_id == self.num_stages - 1: - assert data, ( + if self.is_first_stage or self.is_last_stage: + assert data is not None, ( "For the first and the last stage, the data_iter must be set.") else: - assert data is None, ( - "For pipe stages other than the first and the last one, " - "the data_iter must be None.") + data = None + self.data = data self._layers.train() - self.total_loss = None - - minibatch_cmds = utils.TrainGenerator(self.accumulate_steps, - self.num_stages, self.stage_id) - self._train(minibatch_cmds) - return self.total_loss - def _train(self, minibatch_cmds): - self._allocate_caches(self.accumulate_steps) - for micro_cmds in minibatch_cmds: - for cmd in micro_cmds: - assert type(cmd) in self._COMMAND_MAP, "unknow cmd: {}".format( - type(cmd)) - self._apply_cmd = MethodType(self._COMMAND_MAP[type(cmd)], self) - self._apply_cmd(**cmd.kwargs) - - def _allreduce_grads(self): - if not self.use_data_parallel: return - fused_allreduce_gradients(list(self._layers.parameters()), self._hcg) + # store total loss of entire batch + self.total_loss = None + self._init_caches(self.accumulate_steps) + startup_steps = self.num_stages - self.stage_id - 1 + forward_steps = 0 + backward_steps = 0 + + # forward + while (forward_steps < self.accumulate_steps): + self._forward(cache_id=forward_steps) + forward_steps += 1 + + # backward + while (backward_steps < self.accumulate_steps): + self._backward(cache_id=backward_steps) + backward_steps += 1 + + # optimizer + self._step() + self.train_loss = self._reduce_final_loss() + return self.train_loss def _forward(self, cache_id): # load data @@ -140,16 +154,17 @@ def _forward(self, cache_id): else: inputs = self.caches['inputs'][cache_id] - self._clear_grads(inputs) outputs = self._layers.forward(inputs) + self._clear_grads(inputs) + self.caches['outputs'][cache_id] = outputs - if self.stage_id == self.num_stages - 1: + if self.is_last_stage: if self._layers._loss_fn is not None: labels = self.caches['labels'][cache_id] outputs = self._layers._loss_fn(outputs, labels) - if self.stage_id == self.num_stages - 1: + if self.is_last_stage: self.current_loss = outputs if isinstance(self.current_loss, paddle.Tensor): if self.total_loss is None: @@ -162,18 +177,17 @@ def _forward(self, cache_id): ] for idx, v in enumerate(self.current_loss): self.total_loss[idx] += v.detach() - if self.use_data_parallel: - self.current_loss = self.current_loss / self._hcg.get_data_parallel_world_size( - ) + if self.accumulate_steps > 1: self.current_loss = self.current_loss / self.accumulate_steps + self.caches['outputs'][cache_id] = self.current_loss.clone() + else: self._send_activations(cache_id) def _backward(self, cache_id): - assert self.optimizer is not None - if self.stage_id == self.num_stages - 1: + if self.is_last_stage: paddle.autograd.backward(self.caches['outputs'][cache_id]) self._send_gradients(cache_id) return @@ -194,92 +208,89 @@ def _backward(self, cache_id): grad_tensors = None if self.stage_id != 0: self._send_gradients(cache_id) self.caches['outputs'][cache_id] = None - #self.caches['backward_tensors'][cache_id] = None - def _get_data(self): - if self.use_model_parallel: - mp_rank = self._hcg.get_model_parallel_rank() + def _broadcast_data(self, data): + if isinstance(data, paddle.Tensor): + paddle.distributed.broadcast( + data, + src=self._hcg.get_model_parallel_group_src_rank(), + group=self._hcg.get_model_parallel_group()) else: - mp_rank = 0 - - # mp rank 0 loads the data and broadcat it to others. - data = self.data - if self.use_model_parallel and (self.stage_id == 0 or - self.stage_id == self.num_stages - 1): - assert isinstance(data, (tuple, paddle.Tensor)) - if isinstance(data, paddle.Tensor): + for d in data: + assert isinstance(d, paddle.Tensor) paddle.distributed.broadcast( - data, + d, src=self._hcg.get_model_parallel_group_src_rank(), group=self._hcg.get_model_parallel_group()) - else: - data = [] - for d in self.data: - assert isinstance(d, paddle.Tensor) - paddle.distributed.broadcast( - d, - src=self._hcg.get_model_parallel_group_src_rank(), - group=self._hcg.get_model_parallel_group()) - data.append(d) - data = tuple(data) return data def _load_micro_batch(self, cache_id): - inputs = self._get_data() - - if self.stage_id == 0: - data = None - #if isinstance(inputs[0], paddle.Tensor): - if len(inputs) == 1: - assert isinstance(inputs[0], paddle.Tensor) - data = inputs[0].clone().detach() - #data.stop_gradient = not is_float_tensor(data) - data.stop_gradient = True + inputs = self.data + begin = cache_id * self.micro_batch_size + end = begin + self.micro_batch_size + + if self.is_first_stage: + assert len(inputs) == 2, "length of input should be 2" + if self.use_model_parallel: + inputs[0] = self._broadcast_data(inputs[0]) + if isinstance(inputs[0], tuple): + batch_size = inputs[0][0].shape[0] + assert self.micro_batch_size * self.accumulate_steps == batch_size, ( + "batch_size needs to be divisible by micro_batch_size. Currently, " + "batch_size = %d, micro_batch_size = %d, accumulate_steps = %d." + % + (batch_size, self.micro_batch_size, self.accumulate_steps)) + data = [ + input[begin:end, :].clone().detach() for input in inputs[0] + ] + self.caches['inputs'][cache_id] = tuple(data) + else: + batch_size = inputs[0].shape[0] + assert self.micro_batch_size * self.accumulate_steps == batch_size + self.caches['inputs'][cache_id] = inputs[0][begin:end, :].clone( + ).detach() + elif self.is_last_stage: + assert len(inputs) == 2, "length of input should be 2" + if self.use_model_parallel: + inputs[1] = self._broadcast_data(inputs[1]) + if isinstance(inputs[1], tuple): + batch_size = inputs[1][0].shape[0] + assert self.micro_batch_size * self.accumulate_steps == batch_size + data = [ + input[begin:end, :].clone().detach() for input in inputs[1] + ] + self.caches['labels'][cache_id] = tuple(data) else: - assert isinstance(inputs, tuple) - data = [] - for d in inputs: - assert isinstance(d, paddle.Tensor) - i = d.clone().detach() - #i.stop_gradient = not is_float_tensor(i) - i.stop_gradient = True - data.append(i) - data = tuple(data) - self.caches['inputs'][cache_id] = data - - if self.stage_id == self.num_stages - 1: - labels = None - #if isinstance(inputs[1], paddle.Tensor): - if len(inputs) == 1: - assert isinstance(inputs[0], paddle.Tensor) - labels = inputs[0] - elif isinstance(inputs, tuple): - labels = [] - for label in inputs: - assert isinstance(label, paddle.Tensor) - label = label.detach() - labels.append(label) - labels = tuple(labels) - self.caches['labels'][cache_id] = labels + batch_size = inputs[1].shape[0] + assert self.micro_batch_size * self.accumulate_steps == batch_size + self.caches['labels'][cache_id] = inputs[1][begin:end, :].clone( + ).detach() + else: + # No data input is required for other stages + inputs = None def _send_meta(self, data, peer): - """ - % type (0: tensor, 1: tuple) - % num_tensors if type=tuple - foreach tensor: - % ndims - % shape - """ if isinstance(data, paddle.Tensor): tensor_type = paddle.to_tensor([0]) + # send tensor type paddle.distributed.send( tensor_type, peer, use_calc_stream=True, group=self.pp_group) + + # send len(shape) dims = paddle.to_tensor(len(data.shape)) paddle.distributed.send( dims, peer, use_calc_stream=True, group=self.pp_group) + + # send shape shape = paddle.to_tensor(data.shape) paddle.distributed.send( shape, peer, use_calc_stream=True, group=self.pp_group) + + # send dtype + dtype = paddle.to_tensor(paddle_2_number(data.dtype)) + paddle.distributed.send( + dtype, peer, use_calc_stream=True, group=self.pp_group) + elif isinstance(data, tuple): tensor_type = paddle.to_tensor([1]) paddle.distributed.send( @@ -289,48 +300,73 @@ def _send_meta(self, data, peer): nums, peer, use_calc_stream=True, group=self.pp_group) for idx, d in enumerate(data): assert isinstance(d, paddle.Tensor) + # send len(shape) dims = paddle.to_tensor(len(d.shape)) paddle.distributed.send( dims, peer, use_calc_stream=True, group=self.pp_group) + + # send shape shape = paddle.to_tensor(d.shape) paddle.distributed.send( shape, peer, use_calc_stream=True, group=self.pp_group) + # send dtype + dtype = paddle.to_tensor(paddle_2_number(d.dtype)) + paddle.distributed.send( + dtype, peer, use_calc_stream=True, group=self.pp_group) + def _recv_meta(self, peer): tensor_type = paddle.to_tensor([0]) paddle.distributed.recv( tensor_type, peer, use_calc_stream=True, group=self.pp_group) - tensor_type = tensor_type.numpy()[0] + tensor_type = tensor_type.item() if tensor_type == 0: + # recv len(shape) dims = paddle.to_tensor([0]) paddle.distributed.recv( dims, peer, use_calc_stream=True, group=self.pp_group) - dims = dims.numpy()[0] + dims = dims.item() + + # recv shape shape = paddle.to_tensor([0] * dims) paddle.distributed.recv( shape, peer, use_calc_stream=True, group=self.pp_group) shape = shape.numpy().tolist() - return self._allocate_buffer( - shape, dtype="float32", num_caches=1)[0] + + # recv dtype + dtype = paddle.to_tensor([0]) + paddle.distributed.recv( + dtype, peer, use_calc_stream=True, group=self.pp_group) + return self._allocate_cache( + shape, dtype=number_2_dtype(dtype.item()), num_caches=1)[0] elif tensor_type == 1: num = paddle.to_tensor([0]) paddle.distributed.recv( num, peer, use_calc_stream=True, group=self.pp_group) - num = num.numpy()[0] + num = num.item() shapes = [] + dtypes = [] for i in range(num): + # recv len(shape) dims = paddle.to_tensor([0]) paddle.distributed.recv( dims, peer, use_calc_stream=True, group=self.pp_group) - dims = dims.numpy()[0] + + # recv shape + dims = dims.item() shape = paddle.to_tensor([0] * dims) paddle.distributed.recv( shape, peer, use_calc_stream=True, group=self.pp_group) shapes.append(shape.numpy().tolist()) - dtypes = ["float32"] * len(shapes) - caches = self._allocate_buffers(shapes, dtypes, num_caches=1)[0] + # recv dtype + dtype = paddle.to_tensor([0]) + paddle.distributed.recv( + dtype, peer, use_calc_stream=True, group=self.pp_group) + dtypes.append(number_2_dtype(dtype.item())) + + caches = self._allocate_caches(shapes, dtypes, num_caches=1)[0] caches = tuple(caches) return caches @@ -357,7 +393,6 @@ def _send_activations(self, cache_id): def _send_gradients(self, cache_id): inputs = self.caches['inputs'][cache_id] - if isinstance(inputs, paddle.Tensor): assert inputs.grad is not None paddle.distributed.send( @@ -371,7 +406,6 @@ def _send_gradients(self, cache_id): if not is_float_tensor(d): assert d.grad is None continue - assert d.grad is not None paddle.distributed.send( d.grad, self.prev_stage_id, @@ -381,8 +415,6 @@ def _send_gradients(self, cache_id): def _recv_activations(self, cache_id): inputs = None - - # Allocate the buffer if necessary if self.recv_cache is None: self.recv_cache = self._recv_meta(self.prev_stage_id) @@ -419,14 +451,16 @@ def _recv_gradients(self, cache_id): if self.grad_tensors is None: if isinstance(outputs, paddle.Tensor): s = list(outputs.shape) - dtype = 'float16' if self.use_amp else "float32" - self.grad_tensors = self._allocate_buffer( - s, dtype, num_buffers=1)[0] + dtype = get_tensor_dtype(outputs.dtype) + self.grad_tensors = self._allocate_cache( + s, dtype, num_caches=1)[0] else: sizes = [list(d.shape) for d in outputs if is_float_tensor(d)] - dtypes = ['float16'] * len( - sizes) if self.use_amp else ['float32'] * len(sizes) - self.grad_tensors = self._allocate_buffers( + dtypes = [ + get_tensor_dtype(d.dtype) for d in outputs + if is_float_tensor(d) + ] + self.grad_tensors = self._allocate_caches( sizes, dtypes, num_caches=1)[0] if isinstance(self.grad_tensors, paddle.Tensor): @@ -445,9 +479,10 @@ def _recv_gradients(self, cache_id): group=self.pp_group) def _step(self): - self._allreduce_grads() self.optimizer.step() - self.optimizer.clear_gradients() + self.optimizer.clear_grad() + if self.lr_scheduler: + self.lr_scheduler.step() def _clear_grads(self, inputs): if isinstance(inputs, paddle.Tensor): @@ -461,7 +496,7 @@ def _clear_grads(self, inputs): def _allocate_zeros(self, shape, dtype): return paddle.zeros(shape, dtype) - def _allocate_buffer(self, shape, dtype, num_caches=-1): + def _allocate_cache(self, shape, dtype, num_caches=-1): caches = [] if num_caches == -1: num_caches = self.num_caches @@ -469,7 +504,7 @@ def _allocate_buffer(self, shape, dtype, num_caches=-1): caches.append(self._allocate_zeros(shape, dtype)) return caches - def _allocate_buffers(self, shapes, dtypes, num_caches=-1): + def _allocate_caches(self, shapes, dtypes, num_caches=-1): caches = [] if num_caches == -1: num_caches = self.num_caches @@ -488,11 +523,5 @@ def load_state_dict(self, model_path): state_dict = paddle.load(self.model_path) self._layers.set_state_dict(state_dict) - _COMMAND_MAP = { - utils.Optimize: _step, - utils.Forward: _forward, - utils.Backward: _backward, - } - def forward(self, *inputs, **kwargs): raise RuntimeError("Call train_batch for pipeline instead of forward.") diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py index e5c5709f98d95..8c204820b1661 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py @@ -14,20 +14,51 @@ import abc import paddle -from ...utils import hybrid_parallel_util as hp_util +from ...utils import log_util as hp_util __all__ = [] -FLOAT_TYPES = [ - paddle.float16, - paddle.float32, - paddle.float64, -] +FLOAT_TYPE_DICT = { + paddle.float16: "float16", + paddle.float32: "float32", + paddle.float64: "float64", +} + +PADDLE_TO_NUMBER = { + paddle.float16: 0, + paddle.float32: 1, + paddle.float64: 2, + paddle.int32: 3, + paddle.int64: 4 +} + +NUMBER_TO_DTYPE = { + 0: "float16", + 1: "float32", + 2: "float64", + 3: "int32", + 4: "int64" +} def is_float_tensor(tensor): """Is a float tensor""" - return tensor.dtype in FLOAT_TYPES + return tensor.dtype in FLOAT_TYPE_DICT.keys() + + +def get_tensor_dtype(dtype): + assert dtype in FLOAT_TYPE_DICT.keys() + return FLOAT_TYPE_DICT[dtype] + + +def paddle_2_number(dtype): + assert dtype in PADDLE_TO_NUMBER.keys() + return PADDLE_TO_NUMBER[dtype] + + +def number_2_dtype(number): + assert number in NUMBER_TO_DTYPE.keys() + return NUMBER_TO_DTYPE[number] def get_tensor_bytes(tensor): @@ -48,78 +79,3 @@ def get_tensor_bytes(tensor): else: raise ValueError("unknown data type: {}".format(tensor.dtype)) return tensor.numel() * elem_size - - -class Generator(): - def __init__(self, micro_batches, stages, stage_id): - __metaclass__ = abc.ABCMeta - - self.micro_batches = micro_batches - self.stages = stages - self.stage_id = stage_id - self.prev_stage = self.stage_id - 1 - self.next_stage = self.stage_id + 1 - - @abc.abstractmethod - def generate(self): - pass - - def __iter__(self): - self.iter = None - return self - - def __next__(self): - if self.iter is None: - self.iter = self.generate() - return next(self.iter) - - -class TrainGenerator(Generator): - def generate(self): - startup_steps = self.stages - self.stage_id - 1 - cmds = [] - forward_steps = 0 - backward_steps = 0 - #while (forward_steps < startup_steps): - # cmds.append(Forward(cache_id=forward_steps)) - # forward_steps += 1 - #while (forward_steps < self.micro_batches): - # cmds.append(Forward(cache_id=forward_steps)) - # forward_steps += 1 - # cmds.append(Backward(cache_id=backward_steps)) - # backward_steps += 1 - #while (backward_steps < self.micro_batches): - # cmds.append(Backward(cache_id=backward_steps)) - # backward_steps += 1 - #cmds.append(Optimize()) - while (forward_steps < self.micro_batches): - cmds.append(Forward(cache_id=forward_steps)) - forward_steps += 1 - while (backward_steps < self.micro_batches): - cmds.append(Backward(cache_id=backward_steps)) - backward_steps += 1 - cmds.append(Optimize()) - yield cmds - - -class Command: - def __init__(self, **kwargs): - self.name = self.__class__.__name__ - self.kwargs = kwargs - for key, val in kwargs.items(): - setattr(self, key, val) - - def __repr__(self): - return hp_util.call_to_str(self.name, **self.kwargs) - - -class Optimize(Command): - pass - - -class Forward(Command): - pass - - -class Backward(Command): - pass diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index c1a29c050b138..37bcac4957493 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -22,7 +22,8 @@ list(APPEND DIST_TEST_OPS test_gen_nccl_id_op) list(APPEND DIST_TEST_OPS test_parallel_dygraph_unused_variables) list(APPEND DIST_TEST_OPS test_parallel_dygraph_control_flow) list(APPEND DIST_TEST_OPS test_parallel_dygraph_dataparallel) -list(APPEND DIST_TEST_OPS test_parallel_dygraph_pipeline_layer) +list(APPEND DIST_TEST_OPS test_parallel_dygraph_pipeline_parallel) +list(APPEND DIST_TEST_OPS test_parallel_dygraph_tensor_parallel) list(APPEND DIST_TEST_OPS test_parallel_dygraph_mp_layers) set(MIXED_DIST_TEST_OPS ${DIST_TEST_OPS}) #remove distribute unittests. @@ -176,7 +177,8 @@ if ((NOT WITH_GPU) AND (NOT WITH_ROCM)) LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sync_batch_norm) list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_control_flow) list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_dataparallel) - list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_pipeline_layer) + list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_pipeline_parallel) + list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_tensor_parallel) list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_mp_layers) LIST(REMOVE_ITEM TEST_OPS test_imperative_auto_mixed_precision) LIST(REMOVE_ITEM TEST_OPS test_fleet_base_single) @@ -555,7 +557,7 @@ if(WITH_DISTRIBUTE) set(dist_ut_port 20001) foreach(TEST_OP ${DIST_TEST_OPS}) bash_test_modules(${TEST_OP} START_BASH dist_test.sh SERIAL LABELS "RUN_TYPE=EXCLUSIVE" ENVS "PADDLE_DIST_UT_PORT=${dist_ut_port}") - MATH(EXPR dist_ut_port "${dist_ut_port}+40") + MATH(EXPR dist_ut_port "${dist_ut_port}+35") if(dist_ut_port GREATER_EQUAL 22998) message(FATAL_ERROR "available ports have been exhausted:${dist_ut_port}") endif() @@ -863,7 +865,8 @@ if(WITH_DISTRIBUTE AND WITH_GPU AND WITH_NCCL) set_tests_properties(test_parallel_dygraph_dataparallel PROPERTIES TIMEOUT 120) set_tests_properties(test_parallel_dygraph_unused_variables PROPERTIES TIMEOUT 120) set_tests_properties(test_parallel_dygraph_control_flow PROPERTIES TIMEOUT 120) - set_tests_properties(test_parallel_dygraph_pipeline_layer PROPERTIES TIMEOUT 120) + set_tests_properties(test_parallel_dygraph_pipeline_parallel PROPERTIES TIMEOUT 120) + set_tests_properties(test_parallel_dygraph_tensor_parallel PROPERTIES TIMEOUT 200) set_tests_properties(test_parallel_dygraph_mp_layers PROPERTIES TIMEOUT 120) if(${NCCL_VERSION} VERSION_GREATER_EQUAL 2212) set_tests_properties(test_parallel_dygraph_sparse_embedding PROPERTIES TIMEOUT 120) diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_model.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_model.py index 767bf5d57e74a..a9f251f3079ce 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_model.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_mp_model.py @@ -37,6 +37,7 @@ def set_random_seed(seed, dp_id, rank_id): inner_size = 8 output_size = 2 seq_length = 2 +batch_size = 4 class SimpleMPNet(fluid.dygraph.Layer): @@ -130,18 +131,6 @@ def forward(self, x): return x -class TrainDataset(Dataset): - def __init__(self, length): - self.length = length - - def __len__(self): - return self.length - - def __getitem__(self, index): - np_input_data = np.random.randint(0, vocab_size, (seq_length, )) - return np_input_data - - class TestDistMPTraning(unittest.TestCase): def setUp(self): strategy = fleet.DistributedStrategy() @@ -178,20 +167,6 @@ def build_model_optimizer(self): np_fc1 = np.random.random_sample((hidden_size, inner_size)) np_fc2 = np.random.random_sample((inner_size, hidden_size)) - train_data = TrainDataset(length=10000) - - train_batch_sampler = paddle.io.DistributedBatchSampler( - train_data, - batch_size=4, - shuffle=False, - num_replicas=self.data_parallel_size, - rank=dp_id) - train_data_loader = DataLoader( - dataset=train_data, - batch_sampler=train_batch_sampler, - num_workers=0, - return_list=True) - model_a = SimpleMPNet(vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2, mp_id) optimizer_a = self.build_optimizer(model_a) @@ -202,16 +177,17 @@ def build_model_optimizer(self): np_fc1, np_fc2) optimizer_b = self.build_optimizer(model_b) - return model_a, optimizer_a, model_b, optimizer_b, train_data_loader + return model_a, optimizer_a, model_b, optimizer_b def test_mp_model(self): - model_a, optimizer_a, model_b, optimizer_b, train_data_loader = self.build_model_optimizer( + model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( ) - for step, batch in enumerate(train_data_loader): - if step > 5: - return - + for _ in range(5): + np_data = np.random.randint(0, vocab_size, ( + batch_size, + seq_length, )) + batch = paddle.to_tensor(np_data) loss_a = self.train_batch(batch, model_a, optimizer_a, True) loss_b = self.train_batch(batch, model_b, optimizer_b, False) diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py new file mode 100644 index 0000000000000..912849ffbeb71 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py @@ -0,0 +1,120 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import print_function + +import unittest +import paddle +import numpy as np +import random +import paddle +import paddle.distributed as dist +import paddle.distributed.fleet as fleet +from hybrid_parallel_pp_layer import AlexNetPipeDesc, AlexNet + + +def set_random_seed(seed, dp_id, rank_id): + """Set random seed for reproducability.""" + random.seed(seed) + np.random.seed(seed + dp_id) + paddle.seed(seed + dp_id) + + +batch_size = 4 +micro_batch_size = 2 + + +class TestDistPPTraning(unittest.TestCase): + def setUp(self): + strategy = fleet.DistributedStrategy() + self.model_parallel_size = 1 + self.data_parallel_size = 1 + self.pipeline_parallel_size = 2 + strategy.hybrid_configs = { + "dp_degree": self.data_parallel_size, + "mp_degree": self.model_parallel_size, + "pp_degree": self.pipeline_parallel_size, + } + strategy.pipeline_configs = { + "accumulate_steps": batch_size // micro_batch_size, + "micro_batch_size": micro_batch_size + } + fleet.init(is_collective=True, strategy=strategy) + + def test_pp_model(self): + hcg = fleet.get_hybrid_communicate_group() + word_size = hcg.get_model_parallel_world_size() + dp_id = hcg.get_data_parallel_rank() + pp_id = hcg.get_stage_id() + rank_id = dist.get_rank() + set_random_seed(1024, dp_id, rank_id) + + #construct model a + model_a = AlexNet(10) + scheduler_a = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True) + optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, + parameters=model_a.parameters()) + + param_len = len(model_a.parameters()) + + parameters = [] + for param in model_a.parameters(): + parameters.append(param.numpy()) + + # construct model b + model_b = AlexNetPipeDesc(num_stages=self.pipeline_parallel_size) + scheduler_b = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True) + optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, + parameters=model_b.parameters()) + model_b = fleet.distributed_model(model_b) + optimizer_b = fleet.distributed_optimizer(optimizer_b) + + for idx, param in enumerate(model_b.parameters()): + param.set_value(parameters[idx + pp_id * (param_len // 2)]) + + # construct reader + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True) + + for step_id, data in enumerate(train_reader()): + x_data = np.array([x[0] for x in data]).astype('float32').reshape( + batch_size, 1, 28, 28) + y_data = np.array([x[1] for x in data]).astype('int64').reshape( + batch_size, 1) + img = paddle.to_tensor(x_data) + label = paddle.to_tensor(y_data) + img.stop_gradient = True + label.stop_gradient = True + + if step_id >= 5: + return True + + loss_a = model_a(img, label) + loss_a.backward() + optimizer_a.step() + optimizer_a.clear_grad() + scheduler_a.step() + + loss_b = model_b.train_batch([img, label], optimizer_b, scheduler_b) + + print("loss: ", loss_a.numpy(), loss_b.numpy()) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-5) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_embedding.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_embedding.py new file mode 100644 index 0000000000000..d2be0cb80722b --- /dev/null +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_embedding.py @@ -0,0 +1,208 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import print_function + +import unittest +import paddle +import numpy as np +import random +import paddle +import paddle.distributed as dist +import paddle.distributed.fleet as fleet +from paddle.fluid.dygraph.container import Sequential +from paddle.distributed.fleet.meta_parallel import PipelineLayer +from paddle.fluid.dygraph.layers import Layer +import paddle.nn as nn +import paddle.fluid as fluid + + +def set_random_seed(seed, dp_id, rank_id): + """Set random seed for reproducability.""" + random.seed(seed) + np.random.seed(seed + dp_id) + paddle.seed(seed + dp_id) + + +batch_size = 16 +micro_batch_size = 4 +vocab_size = 128 +hidden_size = 8 + + +class SimpleNet(Layer): + def __init__(self): + super(SimpleNet, self).__init__() + self.word_embeddings = nn.Embedding(vocab_size, hidden_size) + + self.softmax_weight = self.create_parameter( + shape=[hidden_size, vocab_size]) + self.softmax_bias = self.create_parameter( + shape=[vocab_size], is_bias=False) + + def forward(self, x1, x2, y1): + x_emb = self.word_embeddings(x1) + fc = fluid.layers.matmul(x_emb, self.softmax_weight) + fc = fluid.layers.elementwise_add(fc, self.softmax_bias) + projection = fluid.layers.reshape(fc, shape=[-1, vocab_size]) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1, soft_label=False) + return loss.mean() + + +class EmbeddingNet(Layer): + def __init__(self): + super(EmbeddingNet, self).__init__() + self.word_embeddings = nn.Embedding(vocab_size, hidden_size) + + def forward(self, args): + x1, x2 = args + x_emb = self.word_embeddings(x1) + return x_emb, x2 + + +class MatmulNet(Layer): + def __init__(self): + super(MatmulNet, self).__init__() + self.softmax_weight = self.create_parameter( + shape=[hidden_size, vocab_size]) + + def forward(self, args): + x1, x2 = args + fc = fluid.layers.matmul(x1, self.softmax_weight) + + return fc, x2 + + +class BiasNet(Layer): + def __init__(self): + super(BiasNet, self).__init__() + self.softmax_bias = self.create_parameter(shape=[vocab_size]) + + def forward(self, args): + fc, x2 = args + fc = fluid.layers.elementwise_add(fc, self.softmax_bias) + projection = fluid.layers.reshape(fc, shape=[-1, vocab_size]) + return projection, x2 + + +class LossNet(Layer): + def __init__(self): + super(LossNet, self).__init__() + + def forward(self, args, y1): + projection, x2 = args + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1[0], soft_label=False) + return loss.mean() + + +class SimpleNetPipe(Layer): + def __init__(self): + super(SimpleNetPipe, self).__init__() + self.features = Sequential(EmbeddingNet(), MatmulNet(), BiasNet()) + + def to_layers(self): + feat = [self.features[i] for i in range(len(self.features))] + return feat + + +class TestDistEmbeddingTraning(unittest.TestCase): + def setUp(self): + strategy = fleet.DistributedStrategy() + self.model_parallel_size = 1 + self.data_parallel_size = 1 + self.pipeline_parallel_size = 2 + strategy.hybrid_configs = { + "dp_degree": self.data_parallel_size, + "mp_degree": self.model_parallel_size, + "pp_degree": self.pipeline_parallel_size, + } + strategy.pipeline_configs = { + "accumulate_steps": batch_size // micro_batch_size, + "micro_batch_size": micro_batch_size + } + fleet.init(is_collective=True, strategy=strategy) + + def test_pp_model(self): + hcg = fleet.get_hybrid_communicate_group() + word_size = hcg.get_model_parallel_world_size() + dp_id = hcg.get_data_parallel_rank() + pp_id = hcg.get_stage_id() + rank_id = dist.get_rank() + set_random_seed(1024, dp_id, rank_id) + + #construct model a + model_a = SimpleNet() + scheduler_a = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) + optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, + parameters=model_a.parameters()) + + init_net = SimpleNetPipe() + model_b = PipelineLayer( + layers=init_net.to_layers(), + num_stages=self.pipeline_parallel_size, + loss_fn=LossNet()) + + scheduler_b = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) + optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, + parameters=model_b.parameters()) + model_b = fleet.distributed_model(model_b) + optimizer_b = fleet.distributed_optimizer(optimizer_b) + + param_len = len(model_a.parameters()) + + parameters = [] + for param in model_a.parameters(): + print(param.name, param.shape) + parameters.append(param.numpy()) + + model_b_params = model_b.parameters() + if pp_id == 0: + model_b_params[0].set_value(parameters[2]) + else: + model_b_params[0].set_value(parameters[0]) + model_b_params[1].set_value(parameters[1]) + + for step in range(5): + x1_data = np.random.randint(0, vocab_size, size=[batch_size, 1]) + x2_data = np.random.randint(0, vocab_size, size=[batch_size, 1]) + y1_data = np.random.randint(0, 10, size=[batch_size, 1]) + + x1 = paddle.to_tensor(x1_data) + x2 = paddle.to_tensor(x2_data) + y1 = paddle.to_tensor(y1_data) + + x1.stop_gradient = True + x2.stop_gradient = True + y1.stop_gradient = True + + loss_a = model_a(x1, x2, y1) + loss_a.backward() + optimizer_a.step() + optimizer_a.clear_grad() + scheduler_a.step() + + loss_b = model_b.train_batch([(x1, x2), (y1, )], optimizer_b, + scheduler_b) + + print("loss", loss_a.numpy(), loss_b.numpy()) + np.testing.assert_allclose(loss_a.numpy(), loss_b.numpy()) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py index 3130cbf458467..b30df0e9a2f21 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py @@ -12,17 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import unittest import numpy as np import os import paddle from paddle.distributed import fleet -import copy from paddle.fluid.dygraph.container import Sequential import paddle.nn as nn from paddle.fluid.dygraph.layers import Layer from paddle.distributed.fleet.meta_parallel import LayerDesc, PipelineLayer import paddle.nn.functional as F -import unittest + + +class ReshapeHelp(Layer): + def __init__(self, shape): + super(ReshapeHelp, self).__init__() + self.shape = shape + + def forward(self, x): + return x.reshape(shape=self.shape) class AlexNet(Layer): @@ -30,7 +38,7 @@ def __init__(self, num_classes=10): super(AlexNet, self).__init__() self.features = Sequential( nn.Conv2D( - 3, 64, kernel_size=11, stride=4, padding=5), + 1, 64, kernel_size=11, stride=4, padding=5), nn.ReLU(), nn.MaxPool2D( kernel_size=2, stride=2), @@ -50,13 +58,14 @@ def __init__(self, num_classes=10): nn.ReLU(), nn.MaxPool2D( kernel_size=2, stride=2), ) + + self.reshape_layer = ReshapeHelp(shape=[-1, 256]) self.classifier = nn.Linear(256, num_classes) self.loss_fn = nn.loss.CrossEntropyLoss() def forward(self, x, y): x = self.features(x) - x.flatten() - + x = self.reshape_layer(x) x = self.classifier(x) return self.loss_fn(x, y) @@ -64,7 +73,7 @@ def forward(self, x, y): class AlexNetPipe(AlexNet): def to_layers(self): feat = [self.features[i] for i in range(len(self.features))] - loss_fn = [lambda x: x.flatten(), self.classifier] + loss_fn = [self.reshape_layer, self.classifier] feat.extend(loss_fn) return feat @@ -74,7 +83,7 @@ def __init__(self, num_classes=10, **kwargs): self.num_classes = num_classes decs = [ LayerDesc( - nn.Conv2D, 3, 64, kernel_size=11, stride=4, padding=5), + nn.Conv2D, 1, 64, kernel_size=11, stride=4, padding=5), LayerDesc(nn.ReLU), LayerDesc( nn.MaxPool2D, kernel_size=2, stride=2), @@ -94,7 +103,8 @@ def __init__(self, num_classes=10, **kwargs): F.relu, LayerDesc( nn.MaxPool2D, kernel_size=2, stride=2), - lambda x: x.flatten(), + LayerDesc( + ReshapeHelp, shape=[-1, 256]), LayerDesc(nn.Linear, 256, self.num_classes), # classifier ] super(AlexNetPipeDesc, self).__init__( @@ -104,24 +114,24 @@ def __init__(self, num_classes=10, **kwargs): class TestPipeLayerAPI(unittest.TestCase): def setUp(self): strategy = fleet.DistributedStrategy() - self.model_parallel_size = 2 + self.pipeline_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": self.model_parallel_size + "pp_degree": self.pipeline_parallel_size } fleet.init(is_collective=True, strategy=strategy) self.hcg = fleet.get_hybrid_communicate_group() def test_pipelayer_desc(self): - pipe_model = AlexNetPipeDesc(num_stages=self.model_parallel_size) + pipe_model = AlexNetPipeDesc(num_stages=self.pipeline_parallel_size) np.testing.assert_array_equal(len(pipe_model.parameters()), 6) def test_pipelayer_sequential(self): init_net = AlexNetPipe() pipe_model = PipelineLayer( layers=init_net.to_layers(), - num_stages=self.model_parallel_size, + num_stages=self.pipeline_parallel_size, loss_fn=nn.CrossEntropyLoss()) stage_id = self.hcg.get_stage_id() init_parameters = init_net.parameters() diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_model.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_model.py deleted file mode 100644 index 9b9283a1a9b6e..0000000000000 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_model.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import division -from __future__ import print_function - -import paddle -import numpy as np -import random -import paddle.distributed as dist -import paddle.fluid as fluid -import paddle.distributed.fleet as fleet -from paddle.io import DataLoader, Dataset -import unittest - - -def set_random_seed(seed, dp_id, rank_id): - """Set random seed for reproducability.""" - random.seed(seed) - np.random.seed(seed + dp_id) - paddle.seed(seed + rank_id) - - -HIDDEN_DIM = 32 -LAYERS = 8 - - -def sequential_model(): - model = paddle.nn.Sequential( - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, HIDDEN_DIM), - paddle.nn.Linear(HIDDEN_DIM, 1), ) - return model - - -class TestDistPPTraning(unittest.TestCase): - def setUp(self): - strategy = fleet.DistributedStrategy() - self.model_parallel_size = 1 - self.data_parallel_size = 1 - self.pipeline_parallel_size = 2 - strategy.hybrid_configs = { - "dp_degree": self.data_parallel_size, - "mp_degree": self.model_parallel_size, - "pp_degree": self.pipeline_parallel_size, - } - strategy.pipeline_configs = {"accumulate_steps": 2} - paddle.distributed.init_parallel_env() - fleet.init(is_collective=True, strategy=strategy) - - def test_mp_model(self): - batch_input = paddle.randn(shape=(1, HIDDEN_DIM), dtype="float32") - pipe_model = sequential_model() - sgd = paddle.optimizer.SGD(learning_rate=0.0003, parameters=[]) - pipe_model = paddle.distributed.fleet.distributed_model(pipe_model) - - if pipe_model.stage_id == 0 or pipe_model.stage_id == 1: - pipe_input = batch_input.clone().detach() - pipe_input = paddle.cast(pipe_input, 'float32') - - def data_gen(): - gen = True - while gen: - yield [pipe_input, 0] - gen = False - - loader = paddle.io.DataLoader.from_generator(capacity=5) - loader.set_batch_generator(data_gen) - data_iter = iter(loader) - else: - data_iter = None - return True - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py index 5491b451368c8..f3cd97ee1ec86 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py @@ -17,8 +17,11 @@ import unittest import time import paddle.fluid as fluid +import copy +import os +import subprocess -from paddle.distributed.utils import find_free_ports, watch_local_trainers, get_cluster, start_local_trainers +from paddle.distributed.utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc def get_cluster_from_args(selected_gpus): @@ -46,6 +49,55 @@ def get_gpus(selected_gpus): return selected_gpus +def start_local_trainers(cluster, + pod, + training_script, + training_script_args, + log_dir=None): + current_env = copy.copy(os.environ.copy()) + #paddle broadcast ncclUniqueId use socket, and + #proxy maybe make trainers unreachable, so delete them. + #if we set them to "", grpc will log error message "bad uri" + #so just delete them. + current_env.pop("http_proxy", None) + current_env.pop("https_proxy", None) + + procs = [] + for t in pod.trainers: + proc_env = { + "FLAGS_selected_gpus": "%s" % ",".join([str(g) for g in t.gpus]), + "PADDLE_TRAINER_ID": "%d" % t.rank, + "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, + "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + } + + current_env.update(proc_env) + + print("trainer proc env:{}".format(current_env)) + + if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': + cmd = "python -m coverage run --branch -p " + training_script + else: + cmd = "python -u " + training_script + + print("start trainer proc:{} env:{}".format(cmd, proc_env)) + + fn = None + + proc = subprocess.Popen(cmd.split(" "), env=current_env) + + tp = TrainerProc() + tp.proc = proc + tp.rank = t.rank + tp.log_fn = fn + tp.cmd = cmd + + procs.append(tp) + + return procs + + class TestMultipleGpus(unittest.TestCase): def run_mnist_2gpu(self, target_file_name): if not fluid.core.is_compiled_with_cuda( diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_pipeline_layer.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_pipeline_parallel.py similarity index 89% rename from python/paddle/fluid/tests/unittests/test_parallel_dygraph_pipeline_layer.py rename to python/paddle/fluid/tests/unittests/test_parallel_dygraph_pipeline_parallel.py index f3b89d694f70b..1d06e168208b2 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_pipeline_layer.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_pipeline_parallel.py @@ -24,6 +24,9 @@ class TestHybridPipeParallel(TestMultipleGpus): def test_hybrid_parallel_pp_layer(self): self.run_mnist_2gpu('hybrid_parallel_pp_layer.py') + def test_hybrid_parallel_pp_tuple_inputs(self): + self.run_mnist_2gpu('hybrid_parallel_pp_embedding.py') + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_hybrid_parallel.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_tensor_parallel.py similarity index 100% rename from python/paddle/fluid/tests/unittests/test_parallel_dygraph_hybrid_parallel.py rename to python/paddle/fluid/tests/unittests/test_parallel_dygraph_tensor_parallel.py diff --git a/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py b/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py index 7f8294ad0efe7..f62e160673f8d 100644 --- a/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py +++ b/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py @@ -22,7 +22,7 @@ class TestPipelineParallel(TestMultipleGpus): def test_pipeline_parallel(self): - self.run_mnist_2gpu('hybrid_parallel_pp_model.py') + self.run_mnist_2gpu('hybrid_parallel_pp_alexnet.py') if __name__ == "__main__": From 7766721ad39fa76998a8213fcd501c208a7dd48c Mon Sep 17 00:00:00 2001 From: wenbin Date: Mon, 31 May 2021 10:54:03 +0800 Subject: [PATCH 070/156] disable conv plugin in TRT old versions (#33198) --- .../tensorrt/convert/activation_op.cc | 5 -- .../tensorrt/convert/affine_channel_op.cc | 10 --- .../tensorrt/convert/elementwise_op.cc | 4 -- paddle/fluid/inference/tensorrt/op_teller.cc | 21 ++++++ .../ir/inference/test_trt_conv_pass.py | 65 +++++++++++++++++++ 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/activation_op.cc b/paddle/fluid/inference/tensorrt/convert/activation_op.cc index 9244b9af0bbd6..e6a0ecf4aecec 100644 --- a/paddle/fluid/inference/tensorrt/convert/activation_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/activation_op.cc @@ -52,11 +52,6 @@ class ActivationOpConverter : public OpConverter { engine_->GetITensor(op_desc.Input("X")[0]); auto op_pair = ops.find(op_type_); - if (op_pair == ops.end()) { - PADDLE_THROW(platform::errors::Fatal( - "Wrong activation op type, the trt do not support the %s act type.", - op_type_)); - } nvinfer1::IActivationLayer* layer = TRT_ENGINE_ADD_LAYER( engine_, Activation, *const_cast(input_tensor), diff --git a/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc b/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc index 813342c08483b..eba67c3c098ca 100644 --- a/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/affine_channel_op.cc @@ -55,16 +55,6 @@ class AffineChannelOpConverter : public OpConverter { auto* bias_t = bias_v->GetMutable(); float* bias_ptr = engine_->GetWeightCPUData(bias_name, bias_t, false); - auto data_layout = framework::StringToDataLayout( - BOOST_GET_CONST(std::string, op_desc.GetAttr("data_layout"))); - - PADDLE_ENFORCE_EQ( - data_layout, framework::DataLayout::kNCHW, - platform::errors::InvalidArgument( - "TensorRT affine channel converter can only convert NCHW format. " - "Other format should be run in fluid mode. Report a bug on github " - "issue if you see this line.")); - // tensorrt scalend layer only support spatial dims >= 2, // so nhwc is not availabe (spatial dims == 0) const int channel_axis = engine_->with_dynamic_shape(); diff --git a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc index 19d79510547ec..5419933e40736 100644 --- a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc @@ -25,10 +25,6 @@ static bool CheckDims(const nvinfer1::Dims& dims_x, return false; } for (int i = 0; i < dims_x.nbDims; i++) { - // conservative judgment - if (dims_x.d[i] == -1 || dims_y.d[i] == -1) { - return false; - } if (dims_x.d[i] != dims_y.d[i]) { return false; } diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 48c7b7fdd0d79..6db81cefb46a1 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -225,6 +225,27 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, << desc.Output("Output").size() << " output."; return false; } + +// strides > 1 and 'SAME' is only supported by trt7.0 above +#if !IS_TRT_VERSION_GE(7000) + if (op_type == "conv2d" || op_type == "conv2d_fusion" || + op_type == "depthwise_conv2d") { + if (desc.HasAttr("padding_algorithm") && with_dynamic_shape) { + auto padding_algorithm = + BOOST_GET_CONST(std::string, desc.GetAttr("padding_algorithm")); + if (padding_algorithm == "SAME" && desc.HasAttr("strides")) { + const std::vector strides = + BOOST_GET_CONST(std::vector, desc.GetAttr("strides")); + // there is no issue if strides.size() less than 2 + if (strides.size() > 1) { + for (size_t i = 0; i < strides.size(); i++) { + if (strides[i] > 1) return false; + } + } + } + } + } +#endif } if (op_type == "matmul") { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py index ec3955a9ae144..7f613c4765963 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py @@ -161,5 +161,70 @@ def set_params(self): self.use_cudnn = False +class DynamicShapeTensorRTSubgraphPassConvTest(InferencePassTest): + def setUp(self): + self.set_params() + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name="data", shape=[-1, 6, -1, -1], dtype="float32") + conv_out = fluid.layers.conv2d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + stride=self.stride, + act=None) + self.feeds = { + "data": np.random.random([32, 6, 64, 64]).astype("float32"), + } + self.enable_trt = True + self.trt_parameters = DynamicShapeTensorRTSubgraphPassConvTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.dynamic_shape_params = DynamicShapeTensorRTSubgraphPassConvTest.DynamicShapeParam( + { + "conv2d_0.tmp_0": [1, 6, 8, 8], + "data": [1, 6, 8, 8], + "depthwise_conv2d_0.tmp_0": [1, 6, 8, 8] + }, { + "conv2d_0.tmp_0": [32, 6, 64, 64], + "data": [32, 6, 64, 64], + "depthwise_conv2d_0.tmp_0": [32, 6, 64, 64] + }, { + "conv2d_0.tmp_0": [16, 6, 16, 16], + "data": [16, 6, 16, 16], + "depthwise_conv2d_0.tmp_0": [32, 6, 64, 64] + }, False) + self.fetch_list = [conv_out] + + def set_params(self): + self.conv_num_filters = 6 + self.conv_filter_size = 6 + self.conv_groups = 6 + self.conv_padding = 'SAME' + self.use_cudnn = True + self.stride = [2, 2] + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + +class DynamicShapeTensorRTSubgraphPassDepthwiseConvTransposeTest( + DynamicShapeTensorRTSubgraphPassConvTest): + def set_params(self): + self.conv_num_filters = 6 + self.conv_filter_size = 6 + self.conv_groups = 6 + self.conv_padding = 'SAME' + self.use_cudnn = False + self.stride = [2, 2] + + if __name__ == "__main__": unittest.main() From 92a7d11fbe37bd0bdbbb3e0a0bbc1cf32365fc94 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Mon, 31 May 2021 14:52:01 +0800 Subject: [PATCH 071/156] [cherry-pick][CustomOP]Set GLIBCXX_USE_CXX11_ABI=1 to fix potential GCC ABI problem (#33153) (#33185) * Add GLIBCXX_USE_CXX11_ABI flag * fix typo * fix typo --- python/paddle/utils/cpp_extension/cpp_extension.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 6045ac7d1e727..8eefe548b6c6c 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -427,6 +427,12 @@ def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, elif isinstance(cflags, dict): cflags = cflags['cxx'] + # NOTE(Aurelius84): Since Paddle 2.0, we require gcc version > 5.x, + # so we add this flag to ensure the symbol names from user compiled + # shared library have same ABI suffix with core_(no)avx.so. + # See https://stackoverflow.com/questions/34571583/understanding-gcc-5s-glibcxx-use-cxx11-abi-or-the-new-abi + add_compile_flag(['-D_GLIBCXX_USE_CXX11_ABI=1'], cflags) + add_std_without_repeat( cflags, self.compiler.compiler_type, use_std14=False) original_compile(obj, src, ext, cc_args, cflags, pp_opts) From ca0cc8ab94cc34cbc466cd6eb9d60607d6763118 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Tue, 1 Jun 2021 11:05:07 +0800 Subject: [PATCH 072/156] [Cherry-pick][CustomOp]Specify -std=c++14 cflags by default (#33213) (#33227) Cherry-pick (#33213) --- python/paddle/utils/cpp_extension/cpp_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 8eefe548b6c6c..7d6fae3ad7786 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -434,7 +434,7 @@ def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, add_compile_flag(['-D_GLIBCXX_USE_CXX11_ABI=1'], cflags) add_std_without_repeat( - cflags, self.compiler.compiler_type, use_std14=False) + cflags, self.compiler.compiler_type, use_std14=True) original_compile(obj, src, ext, cc_args, cflags, pp_opts) finally: # restore original_compiler From 6fb646065cab1a3df1701d62d9ff3b76dfa17af5 Mon Sep 17 00:00:00 2001 From: WeiXin Date: Tue, 1 Jun 2021 11:19:15 +0800 Subject: [PATCH 073/156] [Cherry-Pick]Set the default value of protocol to 4. (#32904) #33009 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit paddle.save paddle.static.save protocol的默认值改为4（原默认值为2）。 pickle protocol=4相交于protocol=2： protocol=4时保存/加载大于4G的单个numpy.ndarray 等。 protocol=4时保存/加载的速度有明显提升。 Python2 不支持protocol=4（paddle2.1主要支持Python3，不再考虑Python2）。 兼容问题：pickle版本（protocol）会写到文件里面，pickle load的时候会自动识别到protocol，paddle2.1（paddle.save pickle默认版本为2）可以加载paddle2.1.1的模型（paddle.save pickle默认版本为4）。 原始PR：#32904 --- python/paddle/fluid/io.py | 4 ++-- python/paddle/framework/io.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 30baa2aa26cda..30a0b4053e6ff 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -1788,7 +1788,7 @@ def get_tensor(var): @static_only -def save(program, model_path, protocol=2, **configs): +def save(program, model_path, protocol=4, **configs): """ :api_attr: Static Graph @@ -1802,7 +1802,7 @@ def save(program, model_path, protocol=2, **configs): program(Program) : The program to saved. model_path(str): the file prefix to save the program. The format is "dirname/file_prefix". If file_prefix is empty str. A exception will be raised protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5. - Default: 2 + Default: 4 configs(dict, optional) : optional keyword arguments. Returns: diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index de2116cd4382d..1705db50d391a 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -491,7 +491,7 @@ def _save_binary_var(obj, path): format(type(obj))) -def save(obj, path, protocol=2, **configs): +def save(obj, path, protocol=4, **configs): ''' Save an object to the specified path. @@ -512,7 +512,7 @@ def save(obj, path, protocol=2, **configs): path(str) : The path of the object to be saved. If saved in the current directory, the input path string will be used as the file name. protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5. - Default: 2 + Default: 4 **configs(dict, optional): optional keyword arguments. The following options are currently supported: use_binary_format(bool): When the saved object is static graph variable, you can specify ``use_binary_for_var``. If True, save the file in the c++ binary format when saving a single static graph variable; otherwise, save it in pickle format. From 3fe99ad5c1fcd5775945ab56a329572860c66330 Mon Sep 17 00:00:00 2001 From: Qi Li Date: Tue, 1 Jun 2021 23:43:15 +0800 Subject: [PATCH 074/156] [ROCM] add is_compiled_with_rocm api, test=develop (#33043) (#33228) --- python/paddle/__init__.py | 2 ++ python/paddle/device.py | 2 ++ python/paddle/fluid/framework.py | 16 ++++++++++++++++ .../paddle/utils/cpp_extension/cpp_extension.py | 4 ++-- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index ee4dcaa897940..7bac330376c44 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -258,6 +258,7 @@ from .device import set_device # noqa: F401 from .device import get_device # noqa: F401 from .fluid.framework import is_compiled_with_cuda # noqa: F401 +from .fluid.framework import is_compiled_with_rocm # noqa: F401 from .device import is_compiled_with_xpu # noqa: F401 from .device import is_compiled_with_npu # noqa: F401 from .device import XPUPlace # noqa: F401 @@ -384,6 +385,7 @@ 'less_equal', 'triu', 'is_compiled_with_cuda', + 'is_compiled_with_rocm', 'sin', 'dist', 'unbind', diff --git a/python/paddle/device.py b/python/paddle/device.py index 035d240e713fe..85b813a7f51b5 100644 --- a/python/paddle/device.py +++ b/python/paddle/device.py @@ -19,6 +19,7 @@ from paddle.fluid import framework from paddle.fluid.dygraph.parallel import ParallelEnv from paddle.fluid.framework import is_compiled_with_cuda #DEFINE_ALIAS +from paddle.fluid.framework import is_compiled_with_rocm #DEFINE_ALIAS __all__ = [ 'get_cudnn_version', @@ -33,6 +34,7 @@ # 'CUDAPinnedPlace', # 'CUDAPlace', 'is_compiled_with_cuda', + 'is_compiled_with_rocm', 'is_compiled_with_npu' ] diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 3ca16b6667525..bc8a06cb1ed89 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -52,6 +52,7 @@ 'cuda_pinned_places', 'in_dygraph_mode', 'is_compiled_with_cuda', + 'is_compiled_with_rocm', 'is_compiled_with_xpu', 'Variable', 'require_version', @@ -397,6 +398,21 @@ def is_compiled_with_cuda(): return core.is_compiled_with_cuda() +def is_compiled_with_rocm(): + """ + Whether this whl package can be used to run the model on AMD or Hygon GPU(ROCm). + + Returns (bool): `True` if ROCm is currently available, otherwise `False`. + + Examples: + .. code-block:: python + + import paddle + support_gpu = paddle.is_compiled_with_rocm() + """ + return core.is_compiled_with_rocm() + + def cuda_places(device_ids=None): """ **Note**: diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 7d6fae3ad7786..dcaa1ca15e5dc 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -42,10 +42,10 @@ from unittest.mock import Mock _du_build_ext.get_export_symbols = Mock(return_value=None) +CUDA_HOME = find_cuda_home() if core.is_compiled_with_rocm(): ROCM_HOME = find_rocm_home() -else: - CUDA_HOME = find_cuda_home() + CUDA_HOME = ROCM_HOME def setup(**attr): From 8a5a45f8bc6bf4188e6e314646d46ddc477fc0fd Mon Sep 17 00:00:00 2001 From: whs Date: Tue, 1 Jun 2021 23:45:26 +0800 Subject: [PATCH 075/156] Fix cuda kernel launch of grid sampler (#33100) (#33232) --- paddle/fluid/operators/grid_sampler_op.cu | 26 ++++++------ .../unittests/test_bilinear_interp_op.py | 2 + .../tests/unittests/test_grid_sampler_op.py | 42 ++++++++++++++++++- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/paddle/fluid/operators/grid_sampler_op.cu b/paddle/fluid/operators/grid_sampler_op.cu index e9b0a0108afc2..762d14096a5ab 100644 --- a/paddle/fluid/operators/grid_sampler_op.cu +++ b/paddle/fluid/operators/grid_sampler_op.cu @@ -187,7 +187,6 @@ __global__ void grid_sample_cuda_kernel(const int nthreads, int n, int out_c, int out_sC = out_h * out_w; int out_sH = out_w; int out_sW = 1; - CUDA_KERNEL_LOOP(index, nthreads) { const int w = index % out_w; const int h = (index / out_w) % out_h; @@ -199,7 +198,6 @@ __global__ void grid_sample_cuda_kernel(const int nthreads, int n, int out_c, ix = compute_positions(ix, in_w, padding_mode, align_corners); iy = compute_positions(iy, in_h, padding_mode, align_corners); - if (mode == Mode::bilinear) { int ix_nw = static_cast(floor(ix)); int iy_nw = static_cast(floor(iy)); @@ -216,6 +214,7 @@ __global__ void grid_sample_cuda_kernel(const int nthreads, int n, int out_c, T se = (ix - ix_nw) * (iy - iy_nw); auto inp_offset_NC = n * inp_sN; + auto out_ptr_NCHW = output + n * out_sN + h * out_sH + w * out_sW; for (int c = 0; c < out_c; ++c, inp_offset_NC += inp_sC, out_ptr_NCHW += out_sC) { @@ -291,17 +290,17 @@ class GridSampleOpCUDAKernel : public framework::OpKernel { << "; out_w: " << out_w; auto* output = ctx.Output("Output"); auto* output_data = output->mutable_data(ctx.GetPlace()); - - VLOG(3) << "set constant"; + VLOG(3) << "out dims: " << output->dims()[0] << "; " << output->dims()[1] + << "; " << output->dims()[2] << "; " << output->dims()[3]; math::SetConstant()( dev_ctx, output, static_cast(0)); int count = static_cast(n * out_h * out_w); - auto cu_stream = dev_ctx.stream(); - - int block = 512; - int grid_size = (count + block - 1) / block; - grid_sample_cuda_kernel<<>>( + int block_size = 512; + int grid_size = (count + block_size - 1) / block_size; + VLOG(3) << "cuda launch - grid dims: " << grid_size << "; block dims" + << block_size; + grid_sample_cuda_kernel<<>>( count, n, c, out_h, out_w, in_h, in_w, input->data(), grid->data(), output_data, mode, padding_mode, align_corners); } @@ -475,9 +474,12 @@ class GridSampleGradOpCUDAKernel : public framework::OpKernel { int count = static_cast(n * out_h * out_w); auto cu_stream = dev_ctx.stream(); - int block = 512; - int grid_size = (count + block - 1) / block; - grid_sampler_cuda_backward_kernel<<>>( + int block_size = 512; + int grid_size = (count + block_size - 1) / block_size; + VLOG(3) << "cuda launch grad kernel - grid dims: " << grid_size + << "; block dims" << block_size << "; count: " << count; + grid_sampler_cuda_backward_kernel< + T><<>>( count, output_grad->data(), input->data(), grid->data(), n, c, out_h, out_w, in_h, in_w, input_grad->data(), grid_grad_data, mode, padding_mode, align_corners); diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py index 287e85cb271f8..083b671c283a0 100755 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py @@ -19,6 +19,8 @@ from op_test import OpTest import paddle.fluid.core as core import paddle.fluid as fluid +import paddle +paddle.enable_static() def bilinear_interp_np(input, diff --git a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py index bf2f9518fb0c7..1a62f11f597bc 100644 --- a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py +++ b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import paddle import unittest import numpy as np -from op_test import OpTest +import paddle.fluid.core as core +from op_test import OpTest, skip_check_grad_ci +paddle.enable_static() def AffineGrid(theta, grid_shape): @@ -159,7 +162,6 @@ def setUp(self): "padding_mode": self.padding_mode, "mode": self.mode } - # print("X: {}".format(x)) self.outputs = { 'Output': GridSampler(x, grid, self.align_corners, self.mode, self.padding_mode) @@ -236,5 +238,41 @@ def initTestCase(self): self.numeric_grad_delta = 0.0001 +@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass") +class LargeInputCase(TestGridSamplerOp): + def get_places(self): + places = [] + if core.is_compiled_with_cuda(): + places.append(core.CUDAPlace(0)) + return places + + def initTestCase(self): + self.no_need_check_grad = True + self.x_shape = (2, 3, 128, 128) + self.grid_shape = (2, 130, 130, 2) + self.theta_shape = (2, 2, 3) + self.align_corners = False + self.padding_mode = "reflection" + self.mode = "bilinear" + + def test_check_grad_normal(self): + pass + + +@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass") +class Case5(LargeInputCase): + def initTestCase(self): + self.no_need_check_grad = True + self.x_shape = (2, 3, 128, 128) + self.grid_shape = (2, 130, 130, 2) + self.theta_shape = (2, 2, 3) + self.align_corners = True + self.padding_mode = "zeros" + self.mode = "bilinear" + self.use_cudnn = False if core.is_compiled_with_rocm() else True + + if __name__ == "__main__": unittest.main() From 5d8e4395b61929627151f6fd4a607589288a78bf Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Wed, 2 Jun 2021 00:19:11 +0800 Subject: [PATCH 076/156] [Cherry-pick] Fix spawn default nprocs get error (#33215) (#33249) * fix spawn default nprocs get error * polish error message --- python/paddle/distributed/spawn.py | 25 ++++++++++--------- .../test_spawn_and_init_parallel_env.py | 11 +++++++- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/python/paddle/distributed/spawn.py b/python/paddle/distributed/spawn.py index c46672dca09e9..e21f142f10b36 100644 --- a/python/paddle/distributed/spawn.py +++ b/python/paddle/distributed/spawn.py @@ -89,6 +89,18 @@ def _options_valid_check(options): % key) +def _get_default_nprocs(): + device = get_device() + if 'gpu' in device: + return core.get_cuda_device_count() + elif 'xpu' in device: + return core.get_xpu_device_count() + else: + raise RuntimeError( + "`paddle.distributed.spawn` does not support parallel training on device `{}` now.". + format(device)) + + def _get_node_ip(ips): node_ip = None node_ips = [x.strip() for x in ips.split(',')] @@ -448,18 +460,7 @@ def train(print_result=False): # get default nprocs if nprocs == -1: - device = get_device() - if device == 'cpu': - # TODO: not supports cpu parallel now - nprocs = _cpu_num() - elif device == 'gpu': - nprocs = core.get_cuda_device_count() - elif device == 'xpu': - nprocs = core.get_xpu_device_count() - else: - raise ValueError( - "`device` should be a string of `cpu`, 'gpu' or 'xpu', but got {}". - format(device)) + nprocs = _get_default_nprocs() # NOTE(chenweihang): [ why need get cluster info before run? ] # when using `paddle.distributed.spawn` start parallel training, diff --git a/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py b/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py index 6efab81a265ea..14547eca5aca2 100644 --- a/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py +++ b/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py @@ -20,7 +20,7 @@ import paddle import paddle.distributed as dist -from paddle.distributed.spawn import _get_subprocess_env_list, _options_valid_check +from paddle.distributed.spawn import _get_subprocess_env_list, _options_valid_check, _get_default_nprocs from paddle.fluid import core from paddle.fluid.dygraph import parallel_helper @@ -87,6 +87,15 @@ def test_options_valid_check(self): options['error'] = "error" _options_valid_check(options) + def test_get_default_nprocs(self): + paddle.set_device('cpu') + with self.assertRaises(RuntimeError): + nprocs = _get_default_nprocs() + + paddle.set_device('gpu') + nprocs = _get_default_nprocs() + self.assertEqual(nprocs, core.get_cuda_device_count()) + if __name__ == "__main__": unittest.main() From ef6120f32f41250984545c74b0417209aebcf349 Mon Sep 17 00:00:00 2001 From: Qi Li Date: Thu, 3 Jun 2021 13:58:49 +0800 Subject: [PATCH 077/156] [ROCM] fix fused_fc_elementwise_layernorm, test=develop (#33281) (#33299) --- paddle/fluid/platform/cuda_device_function.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/paddle/fluid/platform/cuda_device_function.h b/paddle/fluid/platform/cuda_device_function.h index dde9531e59144..5a86bb46e6ac4 100644 --- a/paddle/fluid/platform/cuda_device_function.h +++ b/paddle/fluid/platform/cuda_device_function.h @@ -32,6 +32,7 @@ namespace platform { #endif inline static int RoundToPowerOfTwo(int dim) { +#ifdef PADDLE_WITH_CUDA if (dim > 512) { return 1024; } else if (dim > 256) { @@ -45,6 +46,17 @@ inline static int RoundToPowerOfTwo(int dim) { } else { return 32; } +#else // HIP results in error or nan if > 256 + if (dim > 128) { + return 256; + } else if (dim > 64) { + return 128; + } else if (dim > 32) { + return 64; + } else { + return 32; + } +#endif } #define CUDA_LAUNCH_KERNEL_BASE(dim, ...) \ From b032b5794fd80d53efda7c2bfba3dd3f7e55c797 Mon Sep 17 00:00:00 2001 From: Qi Li Date: Thu, 3 Jun 2021 13:59:09 +0800 Subject: [PATCH 078/156] [ROCM] update paddle inference cmake, test=develop (#33260) (#33290) --- CMakeLists.txt | 41 ++++++++-------- cmake/configure.cmake | 8 +++ cmake/inference_lib.cmake | 8 ++- cmake/miopen.cmake | 67 ++++++++++++++++++++++++++ paddle/fluid/platform/device_context.h | 9 ++-- paddle/fluid/platform/dynload/miopen.h | 4 +- 6 files changed, 110 insertions(+), 27 deletions(-) create mode 100644 cmake/miopen.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f16c390d8bc7..edb9a46c03ab8 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -283,6 +283,27 @@ if(WITH_GPU) endif() endif() +if(WITH_ROCM) + include(hip) + include(miopen) # set miopen libraries, must before configure +endif(WITH_ROCM) + +if (NOT WITH_ROCM AND WITH_RCCL) + MESSAGE(WARNING + "Disable RCCL when compiling without ROCM. Force WITH_RCCL=OFF.") + set(WITH_RCCL OFF CACHE STRING + "Disable RCCL when compiling without ROCM" FORCE) +endif() + +if(WITH_RCCL) + add_definitions("-DPADDLE_WITH_RCCL") + include(rccl) +else() + if(WITH_ROCM) + MESSAGE(WARNING "If the environment is multi-card, the WITH_RCCL option needs to be turned on, otherwise only a single card can be used.") + endif() +endif() + include(third_party) # download, build, install third_party, Contains about 20+ dependencies include(flags) # set paddle compile flags @@ -307,26 +328,6 @@ include(configure) # add paddle env configuration include_directories("${PADDLE_SOURCE_DIR}") -if(WITH_ROCM) - include(hip) -endif(WITH_ROCM) - -if (NOT WITH_ROCM AND WITH_RCCL) - MESSAGE(WARNING - "Disable RCCL when compiling without ROCM. Force WITH_RCCL=OFF.") - set(WITH_RCCL OFF CACHE STRING - "Disable RCCL when compiling without ROCM" FORCE) -endif() - -if(WITH_RCCL) - add_definitions("-DPADDLE_WITH_RCCL") - include(rccl) -else() - if(WITH_ROCM) - MESSAGE(WARNING "If the environment is multi-card, the WITH_RCCL option needs to be turned on, otherwise only a single card can be used.") - endif() -endif() - if(WITH_NV_JETSON) set(WITH_ARM ON CACHE STRING "Set WITH_ARM=ON when compiling WITH_NV_JETSON=ON." FORCE) endif() diff --git a/cmake/configure.cmake b/cmake/configure.cmake index e7f125269be1f..458ab992c25f3 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -143,6 +143,14 @@ elseif(WITH_ROCM) add_definitions(-DPADDLE_WITH_HIP) add_definitions(-DEIGEN_USE_GPU) add_definitions(-DEIGEN_USE_HIP) + + if(NOT MIOPEN_FOUND) + message(FATAL_ERROR "Paddle needs MIOpen to compile") + endif() + + if(${MIOPEN_VERSION} VERSION_LESS 2090) + message(FATAL_ERROR "Paddle needs MIOPEN >= 2.9 to compile") + endif() else() add_definitions(-DHPPL_STUB_FUNC) list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 9694a7bc59c12..a10b5b231c875 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -323,12 +323,18 @@ function(version version_file) "GIT COMMIT ID: ${PADDLE_GIT_COMMIT}\n" "WITH_MKL: ${WITH_MKL}\n" "WITH_MKLDNN: ${WITH_MKLDNN}\n" - "WITH_GPU: ${WITH_GPU}\n") + "WITH_GPU: ${WITH_GPU}\n" + "WITH_ROCM: ${WITH_ROCM}\n") if(WITH_GPU) file(APPEND ${version_file} "CUDA version: ${CUDA_VERSION}\n" "CUDNN version: v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}\n") endif() + if(WITH_ROCM) + file(APPEND ${version_file} + "HIP version: ${HIP_VERSION}\n" + "MIOpen version: v${MIOPEN_MAJOR_VERSION}.${MIOPEN_MINOR_VERSION}\n") + endif() file(APPEND ${version_file} "CXX compiler version: ${CMAKE_CXX_COMPILER_VERSION}\n") if(TENSORRT_FOUND) file(APPEND ${version_file} diff --git a/cmake/miopen.cmake b/cmake/miopen.cmake new file mode 100644 index 0000000000000..f482f423dc5c1 --- /dev/null +++ b/cmake/miopen.cmake @@ -0,0 +1,67 @@ +if(NOT WITH_ROCM) + return() +endif() + +# Now we don't support ROCm on windows +if(WIN32) + return() +endif() + +set(MIOPEN_ROOT ${ROCM_PATH}/miopen CACHE PATH "MIOPEN ROOT") + +find_path(MIOPEN_INCLUDE_DIR "miopen/miopen.h" + PATHS ${MIOPEN_ROOT} ${MIOPEN_ROOT}/include ${MIOPEN_ROOT}/local/include + $ENV{MIOPEN_ROOT} $ENV{MIOPEN_ROOT}/include $ENV{MIOPEN_ROOT}/local/include + NO_DEFAULT_PATH +) + +get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH) + +find_library(MIOPEN_LIBRARY NAMES "libMIOpen.so" + PATHS ${MIOPEN_ROOT} ${MIOPEN_ROOT}/lib ${MIOPEN_ROOT}/lib64 ${__libpath_hist} + $ENV{MIOPEN_ROOT} $ENV{MIOPEN_ROOT}/lib $ENV{MIOPEN_ROOT}/lib64 + NO_DEFAULT_PATH + DOC "Path to MIOpen library.") + +if(MIOPEN_INCLUDE_DIR AND MIOPEN_LIBRARY) + set(MIOPEN_FOUND ON) +else() + set(MIOPEN_FOUND OFF) +endif() + +macro(find_miopen_version miopen_header_file) + file(READ ${miopen_header_file} MIOPEN_VERSION_FILE_CONTENTS) + get_filename_component(MIOPEN_LIB_PATH ${MIOPEN_LIBRARY} DIRECTORY) + + string(REGEX MATCH "define MIOPEN_VERSION_MAJOR +([0-9]+)" MIOPEN_MAJOR_VERSION + "${MIOPEN_VERSION_FILE_CONTENTS}") + string(REGEX REPLACE "define MIOPEN_VERSION_MAJOR +([0-9]+)" "\\1" + MIOPEN_MAJOR_VERSION "${MIOPEN_MAJOR_VERSION}") + string(REGEX MATCH "define MIOPEN_VERSION_MINOR +([0-9]+)" MIOPEN_MINOR_VERSION + "${MIOPEN_VERSION_FILE_CONTENTS}") + string(REGEX REPLACE "define MIOPEN_VERSION_MINOR +([0-9]+)" "\\1" + MIOPEN_MINOR_VERSION "${MIOPEN_MINOR_VERSION}") + string(REGEX MATCH "define MIOPEN_VERSION_PATCH +([0-9]+)" MIOPEN_PATCH_VERSION + "${MIOPEN_VERSION_FILE_CONTENTS}") + string(REGEX REPLACE "define MIOPEN_VERSION_PATCH +([0-9]+)" "\\1" + MIOPEN_PATCH_VERSION "${MIOPEN_PATCH_VERSION}") + string(REGEX MATCH "define MIOPEN_VERSION_TWEAK +([0-9]+)" MIOPEN_TWEAK_VERSION + "${MIOPEN_VERSION_FILE_CONTENTS}") + string(REGEX REPLACE "define MIOPEN_VERSION_TWEAK +([0-9]+)" "\\1" + MIOPEN_TWEAK_VERSION "${MIOPEN_TWEAK_VERSION}") + + if(NOT MIOPEN_MAJOR_VERSION) + set(MIOPEN_VERSION "???") + else() + add_definitions("-DMIOPEN_MAJOR_VERSION=\"${MIOPEN_MAJOR_VERSION}\"") + math(EXPR MIOPEN_VERSION + "${MIOPEN_MAJOR_VERSION} * 1000 + + ${MIOPEN_MINOR_VERSION} * 10 + ${MIOPEN_PATCH_VERSION}") + message(STATUS "Current MIOpen header is ${MIOPEN_INCLUDE_DIR}/miopen/miopen.h " + "Current MIOpen version is v${MIOPEN_MAJOR_VERSION}.${MIOPEN_MINOR_VERSION}.${MIOPEN_PATCH_VERSION}. ") + endif() +endmacro() + +if(MIOPEN_FOUND) + find_miopen_version(${MIOPEN_INCLUDE_DIR}/miopen/version.h) +endif() diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h index d91e14ec3aa92..a0baf5e81122a 100644 --- a/paddle/fluid/platform/device_context.h +++ b/paddle/fluid/platform/device_context.h @@ -337,15 +337,16 @@ class CUDAContext { PADDLE_ENFORCE_CUDA_SUCCESS(dynload::miopenGetVersion( &miopen_major, &miopen_minor, &miopen_patch)); auto local_miopen_version = - (miopen_major * 1000 + miopen_minor * 100 + miopen_patch) / 100; - auto compile_miopen_version = MIOPEN_VERSION / 100; + (miopen_major * 1000 + miopen_minor * 10 + miopen_patch) / 10; + auto compile_miopen_version = MIOPEN_VERSION / 10; if (local_miopen_version < static_cast(compile_miopen_version)) { LOG_FIRST_N(WARNING, 1) << "WARNING: device: " << place_.device << ". The installed Paddle is compiled with MIOPEN " - << compile_miopen_version / 10 << "." << compile_miopen_version % 10 + << compile_miopen_version / 100 << "." + << compile_miopen_version % 100 << ", but MIOPEN version in your machine is " - << local_miopen_version / 10 << "." << local_miopen_version % 10 + << local_miopen_version / 100 << "." << local_miopen_version % 100 << ", which may cause serious incompatible bug. " << "Please recompile or reinstall Paddle with compatible MIOPEN " "version."; diff --git a/paddle/fluid/platform/dynload/miopen.h b/paddle/fluid/platform/dynload/miopen.h index 5ff4bff4bff65..0298dd55f9a0e 100644 --- a/paddle/fluid/platform/dynload/miopen.h +++ b/paddle/fluid/platform/dynload/miopen.h @@ -21,8 +21,8 @@ limitations under the License. */ #include "paddle/fluid/platform/dynload/dynamic_loader.h" #include "paddle/fluid/platform/port.h" -#define MIOPEN_VERSION \ - (MIOPEN_VERSION_MAJOR * 1000 + MIOPEN_VERSION_MINOR * 100 + \ +#define MIOPEN_VERSION \ + (MIOPEN_VERSION_MAJOR * 1000 + MIOPEN_VERSION_MINOR * 10 + \ MIOPEN_VERSION_PATCH) // NOLINT namespace paddle { From c42ccf14abb6e850a4aa62948e10dc96b6074e38 Mon Sep 17 00:00:00 2001 From: wawltor Date: Fri, 4 Jun 2021 19:11:16 +0800 Subject: [PATCH 079/156] [CherryPick] fix compare ops when broadcast (#33086) * fix compare op in for in the cuda device * fix the paddle compare op for the broadcast --- paddle/fluid/operators/controlflow/compare_op.cc | 8 ++++---- paddle/fluid/operators/controlflow/compare_op.cu | 8 ++++---- .../fluid/tests/unittests/test_compare_op.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/paddle/fluid/operators/controlflow/compare_op.cc b/paddle/fluid/operators/controlflow/compare_op.cc index bf047de86fc21..a03e4165755dd 100644 --- a/paddle/fluid/operators/controlflow/compare_op.cc +++ b/paddle/fluid/operators/controlflow/compare_op.cc @@ -131,18 +131,18 @@ class CompareOp : public framework::OperatorWithKernel { REGISTER_COMPARE_OP(less_than, "Out = X < Y"); REGISTER_COMPARE_KERNEL(less_than, CPU, paddle::operators::LessThanFunctor, - paddle::operators::GreaterEqualFunctor); + paddle::operators::GreaterThanFunctor); REGISTER_COMPARE_OP(less_equal, "Out = X <= Y"); REGISTER_COMPARE_KERNEL(less_equal, CPU, paddle::operators::LessEqualFunctor, - paddle::operators::GreaterThanFunctor); + paddle::operators::GreaterEqualFunctor); REGISTER_COMPARE_OP(greater_than, "Out = X > Y"); REGISTER_COMPARE_KERNEL(greater_than, CPU, paddle::operators::GreaterThanFunctor, - paddle::operators::LessEqualFunctor); + paddle::operators::LessThanFunctor); REGISTER_COMPARE_OP(greater_equal, "Out = X >= Y"); REGISTER_COMPARE_KERNEL(greater_equal, CPU, paddle::operators::GreaterEqualFunctor, - paddle::operators::LessThanFunctor); + paddle::operators::LessEqualFunctor); REGISTER_COMPARE_OP(equal, "Out = X == Y"); REGISTER_COMPARE_KERNEL(equal, CPU, paddle::operators::EqualFunctor, paddle::operators::EqualFunctor); diff --git a/paddle/fluid/operators/controlflow/compare_op.cu b/paddle/fluid/operators/controlflow/compare_op.cu index 3ca700e16e6e7..a60201f9d07d6 100644 --- a/paddle/fluid/operators/controlflow/compare_op.cu +++ b/paddle/fluid/operators/controlflow/compare_op.cu @@ -15,15 +15,15 @@ limitations under the License. */ #include "paddle/fluid/operators/controlflow/compare_op.h" REGISTER_COMPARE_KERNEL(less_than, CUDA, paddle::operators::LessThanFunctor, - paddle::operators::GreaterEqualFunctor); -REGISTER_COMPARE_KERNEL(less_equal, CUDA, paddle::operators::LessEqualFunctor, paddle::operators::GreaterThanFunctor); +REGISTER_COMPARE_KERNEL(less_equal, CUDA, paddle::operators::LessEqualFunctor, + paddle::operators::GreaterEqualFunctor); REGISTER_COMPARE_KERNEL(greater_than, CUDA, paddle::operators::GreaterThanFunctor, - paddle::operators::LessEqualFunctor); + paddle::operators::LessThanFunctor); REGISTER_COMPARE_KERNEL(greater_equal, CUDA, paddle::operators::GreaterEqualFunctor, - paddle::operators::LessThanFunctor); + paddle::operators::LessEqualFunctor); REGISTER_COMPARE_KERNEL(equal, CUDA, paddle::operators::EqualFunctor, paddle::operators::EqualFunctor); REGISTER_COMPARE_KERNEL(not_equal, CUDA, paddle::operators::NotEqualFunctor, diff --git a/python/paddle/fluid/tests/unittests/test_compare_op.py b/python/paddle/fluid/tests/unittests/test_compare_op.py index 8dc80c8931269..a2dd7e49ac4cc 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_op.py @@ -139,6 +139,22 @@ def test_broadcast_api_2(self): fetch_list=[out]) self.assertEqual((res == real_result).all(), True) + def test_broadcast_api_3(self): + paddle.enable_static() + with program_guard(Program(), Program()): + x = paddle.static.data(name='x', shape=[5], dtype='int32') + y = paddle.static.data(name='y', shape=[3, 1], dtype='int32') + op = eval("paddle.%s" % (self.op_type)) + out = op(x, y) + exe = paddle.static.Executor(self.place) + input_x = np.arange(0, 5).reshape((5)).astype(np.int32) + input_y = np.array([5, 3, 2]).reshape((3, 1)).astype(np.int32) + real_result = callback(input_x, input_y) + res, = exe.run(feed={"x": input_x, + "y": input_y}, + fetch_list=[out]) + self.assertEqual((res == real_result).all(), True) + def test_attr_name(self): paddle.enable_static() with program_guard(Program(), Program()): From f17d643079bb4b9cdf32e3eeef27989e28acdbaf Mon Sep 17 00:00:00 2001 From: ceci3 Date: Mon, 7 Jun 2021 13:19:50 +0800 Subject: [PATCH 080/156] Fix syncbn (#32989) (#33321) * fix syncbn --- .../unittests/test_sync_batch_norm_op.py | 67 ++++++++++++++++++- python/paddle/nn/layer/norm.py | 20 ++++-- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py index 13aa7d3d37dd4..47a6d2b811552 100644 --- a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py @@ -248,7 +248,7 @@ def test_convert(self): isinstance(model[idx], paddle.nn.SyncBatchNorm), True) -class TestConvertSyncBatchNormCase2(unittest.TestCase): +class TestConvertSyncBatchNormCast1(unittest.TestCase): def test_convert(self): if not core.is_compiled_with_cuda(): return @@ -277,5 +277,70 @@ def forward(self, x): self.assertEqual(len(compare_model.sublayers()), len(model.sublayers())) +class TestConvertSyncBatchNormCase2(unittest.TestCase): + def test_convert(self): + if not core.is_compiled_with_cuda(): + return + + with fluid.dygraph.guard(fluid.CUDAPlace(0)): + + class SyBNNet(paddle.nn.Layer): + def __init__(self, in_ch=3, out_ch=3, dirate=1): + super(SyBNNet, self).__init__() + self.bn_s1 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( + paddle.nn.BatchNorm3D( + out_ch, + weight_attr=paddle.ParamAttr( + regularizer=paddle.regularizer.L2Decay(0.)))) + self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( + paddle.nn.BatchNorm3D( + out_ch, data_format='NDHWC')) + + def forward(self, x): + x = self.bn_s1(x) + out = paddle.sum(paddle.abs(self.bn_s2(x))) + return out + + class BNNet(paddle.nn.Layer): + def __init__(self, in_ch=3, out_ch=3, dirate=1): + super(BNNet, self).__init__() + self.bn_s1 = paddle.nn.BatchNorm3D( + out_ch, + weight_attr=paddle.ParamAttr( + regularizer=paddle.regularizer.L2Decay(0.))) + self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( + paddle.nn.BatchNorm3D( + out_ch, data_format='NDHWC')) + + def forward(self, x): + x = self.bn_s1(x) + out = paddle.sum(paddle.abs(self.bn_s2(x))) + return out + + bn_model = BNNet() + sybn_model = SyBNNet() + np.random.seed(10) + data = np.random.random([3, 3, 3, 3, 3]).astype('float32') + x = paddle.to_tensor(data) + bn_out = bn_model(x) + sybn_out = sybn_model(x) + self.assertTrue( + np.allclose(bn_out.numpy(), sybn_out.numpy()), + "Output has diff. \n" + "\nBN " + str(bn_out.numpy()) + "\n" + + "Sync BN " + str(sybn_out.numpy())) + + +class TestDygraphSyncBatchNormDataFormatError(unittest.TestCase): + def test_errors(self): + if not core.is_compiled_with_cuda(): + return + + with fluid.dygraph.guard(fluid.CUDAPlace(0)): + my_sync_batch_norm = paddle.nn.SyncBatchNorm(10, data_format='CN') + data = np.random.random([3, 3, 3]).astype('float32') + x = paddle.to_tensor(data) + self.assertRaises(ValueError, my_sync_batch_norm, x) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py index 45640a6598e57..bd39ce30a914e 100644 --- a/python/paddle/nn/layer/norm.py +++ b/python/paddle/nn/layer/norm.py @@ -1057,7 +1057,18 @@ def __init__(self, self).__init__(num_features, momentum, epsilon, weight_attr, bias_attr, data_format, None, name) + def _check_data_format(self): + if self._data_format in ['NCHW', 'NCDHW', 'NC', 'NCL']: + self._data_format = 'NCHW' + elif self._data_format in ["NHWC", "NDHWC", 'NLC']: + self._data_format = 'NHWC' + else: + raise ValueError( + 'expected \'NCDHW\', \'NDHWC\', \'NCL\', \'NLC\', \'NC\', \'NCHW\', \'NHWC\' for data_format' + ) + def forward(self, x): + self._check_data_format() # create output # mean and mean_out share the same memory mean_out = self._mean @@ -1142,11 +1153,12 @@ def convert_sync_batchnorm(cls, layer): """ layer_output = layer if isinstance(layer, _BatchNormBase): - if layer._weight_attr != None and not isinstance(layer._weight_attr, - bool): + if layer._weight_attr != None and not isinstance( + layer._weight_attr, + bool) and layer._weight_attr.name != None: layer._weight_attr.name = layer._weight_attr.name + '_sync' - if layer._bias_attr != None and not isinstance(layer._weight_attr, - bool): + if layer._bias_attr != None and not isinstance( + layer._bias_attr, bool) and layer._bias_attr.name != None: layer._bias_attr.name = layer._bias_attr.name + '_sync' layer_output = SyncBatchNorm(layer._num_features, layer._momentum, From d52251450bcfb04c1f6fdb2b0b14c46d6f2814f7 Mon Sep 17 00:00:00 2001 From: wenbin Date: Mon, 7 Jun 2021 20:55:02 +0800 Subject: [PATCH 081/156] Fix inference prepare data (#33370) --- paddle/fluid/framework/operator.cc | 7 +++- .../fluid/inference/api/analysis_predictor.cc | 39 +++++++++++++++++++ .../ir/inference/test_trt_conv_pass.py | 2 +- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 1e26dab629016..ac4d5a97cf7de 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -1525,7 +1525,12 @@ Scope* OperatorWithKernel::PrepareData( // the rest iterations to save the elapsed time. // We do not support skipping PrepareData in while block, because the Op's // input may be changed by subsequent Ops, which may cause an error. - if (pre_scope_ == &scope && new_scope == nullptr) { + + // For inference, ops that behind conditional branch aren't supported well, + // so disable prepare optimization conservatively. + bool force_prepare_data = HasAttr("inference_force_prepare_data") && + Attr("inference_force_prepare_data"); + if (pre_scope_ == &scope && new_scope == nullptr && !force_prepare_data) { need_prepare_data_ = false; } diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 89c8c7902bac9..e49b33da9c74b 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -270,7 +270,46 @@ bool AnalysisPredictor::CreateExecutor() { executor_.reset(new paddle::framework::NaiveExecutor(place_)); return true; } + +static bool IsPrepareDataOptTargetOp(framework::OpDesc *op) { + // here is prepare data optimization related bad cases: + // let's assume an op behind conditional_block and if conditional_block + // chooses branch 1, the op need to call prepare data. else the op don't need + // to call prepare data. In running, if predictor chooses branch 2, then + // optimization takes effect, later issue is followed if predictor chooses + // branch 1, because the op lost chance to prepare data. + std::vector op_type = {"conditional_block_infer", + "select_input"}; + for (const auto &type : op_type) { + if (op->Type() == type) { + return true; + } + } + return false; +} + +static void DisablePrepareDataOpt( + std::shared_ptr inference_program, int block, + bool pre_disable_opt) { + bool disable_opt = false; + auto &infer_block = inference_program->Block(block); + for (auto *op : infer_block.AllOps()) { + if (disable_opt || pre_disable_opt) { + op->SetAttr("inference_force_prepare_data", true); + } + if (op->HasAttr("sub_block")) { + int blockID = op->GetBlockAttrId("sub_block"); + DisablePrepareDataOpt(inference_program, blockID, + disable_opt || pre_disable_opt); + } + // disable prepare data if unfriendly op is found + disable_opt = IsPrepareDataOptTargetOp(op); + } +} + bool AnalysisPredictor::PrepareExecutor() { + DisablePrepareDataOpt(inference_program_, 0, false); + executor_->Prepare(sub_scope_, *inference_program_, 0, config_.use_feed_fetch_ops_); diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py index 7f613c4765963..adbb89523aacb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py @@ -195,7 +195,7 @@ def setUp(self): }, { "conv2d_0.tmp_0": [16, 6, 16, 16], "data": [16, 6, 16, 16], - "depthwise_conv2d_0.tmp_0": [32, 6, 64, 64] + "depthwise_conv2d_0.tmp_0": [16, 6, 16, 16] }, False) self.fetch_list = [conv_out] From 3c22b1742cc9778f959c879a661147edb54557fa Mon Sep 17 00:00:00 2001 From: Chen Long <1300851984@qq.com> Date: Tue, 8 Jun 2021 10:47:00 +0800 Subject: [PATCH 082/156] [cherry-pick] Fix code examples #32861 #33395 (#33396) * Fix comments in framework (#32861) * Fix comments in framework * Update framework.py * fix code style Co-authored-by: Wenyu --- python/paddle/framework/framework.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/framework/framework.py b/python/paddle/framework/framework.py index 17eaa82cd8b6a..d5fa45f76884f 100644 --- a/python/paddle/framework/framework.py +++ b/python/paddle/framework/framework.py @@ -87,8 +87,6 @@ def get_default_dtype(): @contextmanager def set_grad_enabled(mode): """ - :api_attr: imperative - Create a context which enables or disables dygraph gradient calculation. Args: @@ -96,11 +94,13 @@ def set_grad_enabled(mode): Examples: .. code-block:: python + + import paddle x = paddle.ones([3, 2]) x.stop_gradient = False - with torch.set_grad_enabled(False): + with paddle.set_grad_enabled(False): y = x * 2 - with torch.set_grad_enabled(True): + with paddle.set_grad_enabled(True): z = x * 2 print(y.stop_gradient) # True print(z.stop_gradient) # False From ccabafa6df9d98103f675bf4733039a8cfa66931 Mon Sep 17 00:00:00 2001 From: TeslaZhao Date: Tue, 8 Jun 2021 11:10:48 +0800 Subject: [PATCH 083/156] OP:strided_slice_op supports bool type inputs (#33373) (#33393) * Fix two english api documents, transpose and strided_slice * OP:strided_slice_op supports bool type inputs --- paddle/fluid/operators/strided_slice_op.cc | 2 + paddle/fluid/operators/strided_slice_op.cu | 4 +- python/paddle/fluid/layers/nn.py | 4 +- .../tests/unittests/test_strided_slice_op.py | 65 +++++++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/strided_slice_op.cc b/paddle/fluid/operators/strided_slice_op.cc index e49476e4dc7d4..effacc7591de8 100644 --- a/paddle/fluid/operators/strided_slice_op.cc +++ b/paddle/fluid/operators/strided_slice_op.cc @@ -324,6 +324,7 @@ REGISTER_OPERATOR(strided_slice_grad, ops::StridedSliceOpGrad, REGISTER_OP_CPU_KERNEL( strided_slice, + ops::StridedSliceKernel, ops::StridedSliceKernel, ops::StridedSliceKernel, ops::StridedSliceKernel, @@ -335,6 +336,7 @@ REGISTER_OP_CPU_KERNEL( REGISTER_OP_CPU_KERNEL( strided_slice_grad, + ops::StridedSliceGradKernel, ops::StridedSliceGradKernel, ops::StridedSliceGradKernel, ops::StridedSliceGradKernel, diff --git a/paddle/fluid/operators/strided_slice_op.cu b/paddle/fluid/operators/strided_slice_op.cu index b85403b1c5bb8..edf843bb3eeeb 100644 --- a/paddle/fluid/operators/strided_slice_op.cu +++ b/paddle/fluid/operators/strided_slice_op.cu @@ -19,6 +19,7 @@ limitations under the License. */ namespace ops = paddle::operators; REGISTER_OP_CUDA_KERNEL( strided_slice, + ops::StridedSliceKernel, ops::StridedSliceKernel, ops::StridedSliceKernel, ops::StridedSliceKernel, @@ -30,7 +31,8 @@ REGISTER_OP_CUDA_KERNEL( REGISTER_OP_CUDA_KERNEL( strided_slice_grad, - ops::StridedSliceGradKernel, + ops::StridedSliceGradKernel, + ops::StridedSliceGradKernel, ops::StridedSliceGradKernel, ops::StridedSliceGradKernel, ops::StridedSliceGradKernel, diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 9ac314528dc1f..2bac3289d1b64 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -11075,7 +11075,7 @@ def strided_slice(input, axes, starts, ends, strides): Then: result = [ [2], ] Args: - input (Variable): An N-D ``Tensor`` or ``LoDTensor`` . The data type is ``float32``, ``float64``, ``int32`` or ``int64``. + input (Variable): An N-D ``Tensor`` or ``LoDTensor`` . The data type is ``bool``, ``float32``, ``float64``, ``int32`` or ``int64``. axes (list|tuple): The data type is ``int32`` . Axes that `starts` and `ends` apply to. It's optional. If it is not provides, it will be treated as :math:`[0,1,...,len(starts)-1]`. starts (list|tuple|Variable): The data type is ``int32`` . If ``starts`` is a list or tuple, the elements of @@ -11126,7 +11126,7 @@ def strided_slice(input, axes, starts, ends, strides): helper = LayerHelper('strided_slice', **locals()) check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], + ['bool', 'float32', 'float64', 'int32', 'int64'], 'strided_slice') check_type(axes, 'axes', (list, tuple), 'strided_slice') check_type(starts, 'starts', (list, tuple, Variable), 'strided_slice') diff --git a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py index 71550c8f24753..ebf7c01e2cae5 100644 --- a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py @@ -216,6 +216,71 @@ def initTestCase(self): self.infer_flags = [1, 1, 1, 1, 1] +class TestStrideSliceOpBool(TestStrideSliceOp): + def test_check_grad(self): + pass + + +class TestStrideSliceOpBool1D(TestStrideSliceOpBool): + def initTestCase(self): + self.input = np.random.rand(100).astype("bool") + self.axes = [0] + self.starts = [3] + self.ends = [8] + self.strides = [1] + self.infer_flags = [1] + + +class TestStrideSliceOpBool2D(TestStrideSliceOpBool): + def initTestCase(self): + self.input = np.random.rand(10, 10).astype("bool") + self.axes = [0, 1] + self.starts = [1, 0] + self.ends = [2, 2] + self.strides = [1, 1] + self.infer_flags = [1, 1] + + +class TestStrideSliceOpBool3D(TestStrideSliceOpBool): + def initTestCase(self): + self.input = np.random.rand(3, 4, 10).astype("bool") + self.axes = [0, 1, 2] + self.starts = [0, -1, 0] + self.ends = [2, -3, 5] + self.strides = [1, -1, 1] + self.infer_flags = [1, 1, 1] + + +class TestStrideSliceOpBool4D(TestStrideSliceOpBool): + def initTestCase(self): + self.input = np.random.rand(3, 3, 3, 4).astype("bool") + self.axes = [0, 1, 2, 3] + self.starts = [1, 0, 0, 0] + self.ends = [2, 2, 3, 4] + self.strides = [1, 1, 1, 2] + self.infer_flags = [1, 1, 1, 1] + + +class TestStrideSliceOpBool5D(TestStrideSliceOpBool): + def initTestCase(self): + self.input = np.random.rand(3, 3, 3, 4, 5).astype("bool") + self.axes = [0, 1, 2, 3, 4] + self.starts = [1, 0, 0, 0, 0] + self.ends = [2, 2, 3, 4, 4] + self.strides = [1, 1, 1, 1, 1] + self.infer_flags = [1, 1, 1, 1] + + +class TestStrideSliceOpBool6D(TestStrideSliceOpBool): + def initTestCase(self): + self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype("bool") + self.axes = [0, 1, 2, 3, 4, 5] + self.starts = [1, 0, 0, 0, 1, 2] + self.ends = [2, 2, 3, 1, 2, 8] + self.strides = [1, 1, 1, 1, 1, 2] + self.infer_flags = [1, 1, 1, 1, 1] + + class TestStridedSliceOp_starts_ListTensor(OpTest): def setUp(self): self.op_type = "strided_slice" From 0549d4af3c41c6013901a9c584ccac5236a07779 Mon Sep 17 00:00:00 2001 From: Pei Yang Date: Tue, 8 Jun 2021 12:41:25 +0800 Subject: [PATCH 084/156] Cherry pick deconv & jetson single arch (#33387) * fix conv2d_transpose trt bugs (#33242) * fix jetson arch when compiling with single arch (#33269) --- cmake/cuda.cmake | 18 +++++++++++--- .../inference/tensorrt/convert/conv2d_op.cc | 19 +++++++++------ .../ir/inference/test_trt_conv_pass.py | 24 +++++++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index 7f2addb02d36d..59c9070d1ae58 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -95,11 +95,23 @@ function(select_nvcc_arch_flags out_variable) if(${CUDA_ARCH_NAME} STREQUAL "Kepler") set(cuda_arch_bin "30 35") elseif(${CUDA_ARCH_NAME} STREQUAL "Maxwell") - set(cuda_arch_bin "50") + if (WITH_NV_JETSON) + set(cuda_arch_bin "53") + else() + set(cuda_arch_bin "50") + endif() elseif(${CUDA_ARCH_NAME} STREQUAL "Pascal") - set(cuda_arch_bin "60 61") + if (WITH_NV_JETSON) + set(cuda_arch_bin "62") + else() + set(cuda_arch_bin "60 61") + endif() elseif(${CUDA_ARCH_NAME} STREQUAL "Volta") - set(cuda_arch_bin "70") + if (WITH_NV_JETSON) + set(cuda_arch_bin "72") + else() + set(cuda_arch_bin "70") + endif() elseif(${CUDA_ARCH_NAME} STREQUAL "Turing") set(cuda_arch_bin "75") elseif(${CUDA_ARCH_NAME} STREQUAL "Ampere") diff --git a/paddle/fluid/inference/tensorrt/convert/conv2d_op.cc b/paddle/fluid/inference/tensorrt/convert/conv2d_op.cc index 61199724bcfe3..6bbda6bb29aad 100644 --- a/paddle/fluid/inference/tensorrt/convert/conv2d_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/conv2d_op.cc @@ -103,11 +103,18 @@ void ConvertConv2d(TensorRTEngine* engine, const framework::proto::OpDesc& op, TensorRTEngine::Weight bias{nvinfer1::DataType::kFLOAT, static_cast(bias_data), bias_size}; - auto* layer = fadd_layer(const_cast(X), n_output, n_input, - nv_ksize, weight, bias); - PADDLE_ENFORCE_NOT_NULL(layer, - platform::errors::Fatal("TensorRT create conv2d" - " layer error.")); + // In conv2d_transpose and depthwise_conv2d_transpose, + // output channels = filter_dims[1] * groups + auto* layer = (op_desc.Type() == "conv2d_transpose" || + op_desc.Type() == "depthwise_conv2d_transpose") + ? fadd_layer(const_cast(X), + n_input * groups, nv_ksize, weight, bias) + : fadd_layer(const_cast(X), n_output, + nv_ksize, weight, bias); + + PADDLE_ENFORCE_NOT_NULL( + layer, platform::errors::Fatal("TensorRT create conv2d/conv2d_transpose" + " layer failed.")); layer->setStride(nv_strides); layer->setPadding(nv_paddings); layer->setNbGroups(groups); @@ -134,7 +141,6 @@ class Conv2dOpConverter : public OpConverter { ConvertConv2d( engine_, op, scope, test_mode, [&](nvinfer1::ITensor* inputs, int n_output, /* Conv output maps */ - int n_input, /* Conv input maps */ nvinfer1::DimsHW& ksize, TensorRTEngine::Weight& weight, TensorRTEngine::Weight& bias) -> nvinfer1::IConvolutionLayer* { auto* layer = @@ -156,7 +162,6 @@ class Deconv2dOpConverter : public OpConverter { ConvertConv2d( engine_, op, scope, test_mode, [&](nvinfer1::ITensor* inputs, int n_output, /* Deconv input maps */ - int n_input, /* Deconv output maps */ nvinfer1::DimsHW& ksize, TensorRTEngine::Weight& weight, TensorRTEngine::Weight& bias) -> nvinfer1::IDeconvolutionLayer* { auto* layer = diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py index adbb89523aacb..ebbf724d0b4ea 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py @@ -36,6 +36,7 @@ def setUp(self): groups=self.conv_groups, padding=self.conv_padding, bias_attr=False, + use_cudnn=self.use_cudnn, act=None) self.feeds = { "data": np.random.random([1, 6, 64, 64]).astype("float32"), @@ -50,6 +51,7 @@ def set_params(self): self.conv_filter_size = 6 self.conv_groups = 3 self.conv_padding = [1, 1] + self.use_cudnn = True def test_check_output(self): if core.is_compiled_with_cuda(): @@ -65,6 +67,7 @@ def set_params(self): self.conv_filter_size = 6 self.conv_groups = 3 self.conv_padding = 'VALID' + self.use_cudnn = True class TensorRTSubgraphPassConvSamePaddingTest(InferencePassTest): @@ -73,6 +76,7 @@ def set_params(self): self.conv_filter_size = 6 self.conv_groups = 3 self.conv_padding = 'SAME' + self.use_cudnn = True class TensorRTSubgraphPassDepthwiseConvTest(TensorRTSubgraphPassConvTest): @@ -81,6 +85,16 @@ def set_params(self): self.conv_filter_size = 6 self.conv_groups = 6 self.conv_padding = [1, 1] + self.use_cudnn = False + + +class TensorRTSubgraphPassDepthwiseConv2Test(TensorRTSubgraphPassConvTest): + def set_params(self): + self.conv_num_filters = 12 + self.conv_filter_size = 6 + self.conv_groups = 6 + self.conv_padding = [1, 1] + self.use_cudnn = False class TensorRTSubgraphPassConvTransposeTest(InferencePassTest): @@ -151,6 +165,16 @@ def set_params(self): self.use_cudnn = True +class TensorRTSubgraphPassConvTranspose2Test( + TensorRTSubgraphPassConvTransposeTest): + def set_params(self): + self.conv_num_filters = 12 + self.conv_filter_size = 4 + self.conv_groups = 6 + self.conv_padding = [1, 1] + self.use_cudnn = False + + class TensorRTSubgraphPassDepthwiseConvTransposeTest( TensorRTSubgraphPassConvTransposeTest): def set_params(self): From 5e09d67a09a4631ce9a3e09eab0d7aa6a005418d Mon Sep 17 00:00:00 2001 From: Shibo Tao <62922815+T8T9@users.noreply.github.com> Date: Tue, 8 Jun 2021 15:40:19 +0800 Subject: [PATCH 085/156] fix API: normalize_program. test=develop (#33408) * fix: paddle.static.default_main_program. test=develop * add normalize_program to __all__. test=develop --- python/paddle/static/__init__.py | 1 + python/paddle/static/io.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/static/__init__.py b/python/paddle/static/__init__.py index 89da75ae91e40..688bff4a678f2 100644 --- a/python/paddle/static/__init__.py +++ b/python/paddle/static/__init__.py @@ -85,6 +85,7 @@ 'load', 'save_inference_model', 'load_inference_model', + 'normalize_program', 'load_program_state', 'set_program_state', 'cpu_places', diff --git a/python/paddle/static/io.py b/python/paddle/static/io.py index 58e8ebc481d79..a9cae0c14e3b1 100644 --- a/python/paddle/static/io.py +++ b/python/paddle/static/io.py @@ -157,7 +157,7 @@ def normalize_program(program, feed_vars, fetch_vars): exe.run(paddle.static.default_startup_program()) # normalize main program. - program = default_main_program() + program = paddle.static.default_main_program() normalized_program = paddle.static.normalize_program(program, [image], [predict]) """ From bad3bebf8ef48545011e4aaf21568b8e17dc66a7 Mon Sep 17 00:00:00 2001 From: Wangzheee <634486483@qq.com> Date: Tue, 8 Jun 2021 18:47:54 +0800 Subject: [PATCH 086/156] Add trt convert reshape_op in release/2.1.1 (#33372) --- .../fluid/inference/api/analysis_predictor.cc | 1 + .../inference/tensorrt/convert/CMakeLists.txt | 1 + .../inference/tensorrt/convert/op_converter.h | 7 ++ .../inference/tensorrt/convert/reshape_op.cc | 63 ++++++++++ paddle/fluid/inference/tensorrt/op_teller.cc | 17 +++ .../unittests/ir/inference/CMakeLists.txt | 3 +- .../ir/inference/test_trt_reshape_op.py | 109 ++++++++++++++++++ 7 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 paddle/fluid/inference/tensorrt/convert/reshape_op.cc create mode 100644 python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index e49b33da9c74b..ba729fe0492e9 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -1234,6 +1234,7 @@ USE_TRT_CONVERTER(roi_align); USE_TRT_CONVERTER(affine_channel); USE_TRT_CONVERTER(multiclass_nms); USE_TRT_CONVERTER(nearest_interp); +USE_TRT_CONVERTER(reshape); #endif namespace paddle_infer { diff --git a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt index 3820ac5d7cc24..99328e6076891 100644 --- a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt +++ b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt @@ -12,6 +12,7 @@ nv_library(tensorrt_converter affine_channel_op.cc multiclass_nms_op.cc nearest_interp_op.cc + reshape_op.cc DEPS tensorrt_engine tensorrt_plugin operator scope framework_proto op_registry) nv_test(test_op_converter SRCS test_op_converter.cc DEPS diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h index f72ae2c3ec2d7..57a26aec6ebcb 100644 --- a/paddle/fluid/inference/tensorrt/convert/op_converter.h +++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h @@ -127,6 +127,13 @@ class OpConverter { it, platform::errors::Unimplemented("no OpConverter for optype [%s]", op_desc.Type())); } + // reshape2 == reshape + if (op_desc.Type() == "reshape2") { + it = Registry::Global().Lookup("reshape"); + PADDLE_ENFORCE_NOT_NULL( + it, platform::errors::Unimplemented("no OpConverter for optype [%s]", + op_desc.Type())); + } if (!it) { it = Registry::Global().Lookup(op_desc.Type()); } diff --git a/paddle/fluid/inference/tensorrt/convert/reshape_op.cc b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc new file mode 100644 index 0000000000000..3d8c72728c667 --- /dev/null +++ b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc @@ -0,0 +1,63 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/inference/tensorrt/convert/op_converter.h" + +namespace paddle { +namespace framework { +class Scope; +namespace proto { +class OpDesc; +} // namespace proto +} // namespace framework +} // namespace paddle + +namespace paddle { +namespace inference { +namespace tensorrt { + +/* + * ReshapeOp + */ +class ReshapeOpConverter : public OpConverter { + public: + void operator()(const framework::proto::OpDesc& op, + const framework::Scope& scope, bool test_mode) override { + framework::OpDesc op_desc(op, nullptr); + // Declare inputs + auto* input = engine_->GetITensor(op_desc.Input("X")[0]); + const std::vector& shape = + BOOST_GET_CONST(std::vector, op_desc.GetAttr("shape")); + int nbDims_num = shape.size(); + nvinfer1::Dims reshape_dim; + if (engine_->with_dynamic_shape()) { // running the TRT Dynamic Shape mode + reshape_dim.nbDims = nbDims_num; + for (int i = 0; i < nbDims_num; ++i) { + reshape_dim.d[i] = shape[i]; + } + } else { // running the TRT Static Shape mode + reshape_dim.nbDims = nbDims_num - 1; + for (int i = 0; i < nbDims_num - 1; ++i) { + reshape_dim.d[i] = shape[i + 1]; + } + } + auto* layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); + layer->setReshapeDimensions(reshape_dim); + auto output_name = op_desc.Output("Out")[0]; + RreplenishLayerAndOutput(layer, "reshape", {output_name}, test_mode); + } +}; + +} // namespace tensorrt +} // namespace inference +} // namespace paddle + +REGISTER_TRT_OP_CONVERTER(reshape, ReshapeOpConverter); diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 6db81cefb46a1..e7a48013b07f4 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -49,6 +49,10 @@ struct SimpleOpTypeSetTeller : public Teller { #endif #if IS_TRT_VERSION_GE(7130) teller_set.insert("group_norm"); +#endif +#if CUDA_VERSION >= 10200 + teller_set.insert("reshape"); + teller_set.insert("reshape2"); #endif } @@ -654,6 +658,19 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } } + if (op_type == "reshape" || op_type == "reshape2") { + if (!desc.HasAttr("shape") || with_dynamic_shape) { + return false; + // Paddle-TRT does not support the input tensors: Shape and ShapeTensor + } else if (desc.Input("Shape").size() >= 1 || + desc.Input("ShapeTensor").size() >= 1) { + return false; + } else { + std::vector shape = + BOOST_GET_CONST(std::vector, desc.GetAttr("shape")); + if (shape.size() >= nvinfer1::Dims::MAX_DIMS) return false; + } + } if ((*teller)(op_type, desc, use_no_calib_int8)) return true; } return false; diff --git a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt index 8e4c091cd01dd..0f068045e0c09 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt @@ -8,6 +8,7 @@ foreach(TEST_INFERENCE_IR_PASS ${TEST_TRT_IR_PASSES}) endforeach() if(WITH_GPU AND TENSORRT_FOUND) + list(REMOVE_ITEM TEST_TRT_IR_PASSES test_trt_multiclass_nms_op) foreach(target ${TEST_TRT_IR_PASSES}) py_test_modules(${target} MODULES ${target}) endforeach() @@ -32,6 +33,6 @@ if(WITH_GPU AND TENSORRT_FOUND) set_tests_properties(test_trt_subgraph_pass PROPERTIES TIMEOUT 120) set_tests_properties(test_trt_activation_pass PROPERTIES TIMEOUT 120) set_tests_properties(test_trt_conv_pass PROPERTIES TIMEOUT 120) -set_tests_properties(test_trt_multiclass_nms_op PROPERTIES TIMEOUT 200) +#set_tests_properties(test_trt_multiclass_nms_op PROPERTIES TIMEOUT 200) set_tests_properties(test_trt_dynamic_shape PROPERTIES TIMEOUT 120) endif() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py new file mode 100644 index 0000000000000..90a6c482cdbba --- /dev/null +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py @@ -0,0 +1,109 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +from inference_pass_test import InferencePassTest +import paddle.fluid as fluid +import paddle.fluid.core as core +from paddle.fluid.core import PassVersionChecker +from paddle.fluid.core import AnalysisConfig + + +class TRTReshapeTest(InferencePassTest): + def setUp(self): + self.bs = 1 + self.input_shape = [32, 15, 24] + self.reshape = [-1, 8, 20, 72] + self.data_shape = [ + self.bs, self.input_shape[0], self.input_shape[1], + self.input_shape[2] + ] + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32') + reshape_out = self.append_reshape(data, self.reshape) + out = fluid.layers.batch_norm(reshape_out, is_test=True) + self.feeds = { + 'data': np.random.random(self.data_shape).astype('float32'), + } + self.enable_trt = True + self.trt_parameters = TRTReshapeTest.TensorRTParam( + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + self.fetch_list = [out] + + def append_reshape(self, data, reshape): + return fluid.layers.reshape(data, reshape) + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + +class TRTReshapeTest1(TRTReshapeTest): + def setUp(self): + self.bs = 2 + self.input_shape = [23, 13, 24] + self.reshape = [2, 0, -1, 12] + self.data_shape = [ + self.bs, self.input_shape[0], self.input_shape[1], + self.input_shape[2] + ] + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32') + reshape_out = self.append_reshape(data, self.reshape) + out = fluid.layers.batch_norm(reshape_out, is_test=True) + self.feeds = { + 'data': np.random.random(self.data_shape).astype('float32'), + } + self.enable_trt = True + self.trt_parameters = TRTReshapeTest.TensorRTParam( + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + self.fetch_list = [out] + + +class TRTReshapeTest2(TRTReshapeTest): + def setUp(self): + self.bs = 1 + self.input_shape = [14, 48, 27] + self.reshape = [1, 24, 28, 0] + self.data_shape = [ + self.bs, self.input_shape[0], self.input_shape[1], + self.input_shape[2] + ] + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32') + bn_out = fluid.layers.batch_norm(data, is_test=True) + out = self.append_reshape(bn_out, self.reshape) + self.feeds = { + 'data': np.random.random(self.data_shape).astype('float32'), + } + self.enable_trt = True + self.trt_parameters = TRTReshapeTest.TensorRTParam( + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + self.dynamic_shape_params = TRTReshapeTest.DynamicShapeParam({ + 'data': [1, 3, 8, 8] + }, {'data': [5, 100, 100, 100]}, {'data': [1, 3, 16, 16]}, False) + self.fetch_list = [out] + + +if __name__ == "__main__": + unittest.main() From 28a18af023e97831d617594e88327a8c8e7531f0 Mon Sep 17 00:00:00 2001 From: wangguanzhong Date: Wed, 9 Jun 2021 10:42:29 +0800 Subject: [PATCH 087/156] fix output_padding in conv (#33429) --- python/paddle/nn/layer/conv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/nn/layer/conv.py b/python/paddle/nn/layer/conv.py index 2de065d62a4f8..51eab0d1838c9 100644 --- a/python/paddle/nn/layer/conv.py +++ b/python/paddle/nn/layer/conv.py @@ -98,7 +98,7 @@ def __init__(self, 'kernel_size') self._padding = padding self._padding_mode = padding_mode - self.output_padding = output_padding + self._output_padding = output_padding if dims != 1: self._updated_padding, self._padding_algorithm = _update_padding_nd( padding, channel_last, dims) @@ -163,7 +163,7 @@ def extra_repr(self): main_str += ', padding={_padding}' if self._padding_mode is not 'zeros': main_str += ', padding_mode={_padding_mode}' - if self.output_padding != 0: + if self._output_padding != 0: main_str += ', output_padding={_output_padding}' if self._dilation != [1] * len(self._dilation): main_str += ', dilation={_dilation}' @@ -502,7 +502,7 @@ def forward(self, x, output_size=None): self.weight, bias=self.bias, output_size=output_size, - output_padding=self.output_padding, + output_padding=self._output_padding, padding=self._padding, stride=self._stride, dilation=self._dilation, @@ -810,7 +810,7 @@ def __init__(self, def forward(self, x, output_size=None): if output_size is None: - output_padding = self.output_padding + output_padding = self._output_padding else: output_padding = 0 @@ -1139,7 +1139,7 @@ def __init__(self, def forward(self, x, output_size=None): if output_size is None: - output_padding = self.output_padding + output_padding = self._output_padding else: output_padding = 0 From 6385f5eee99bb119d00b1ef2de5c4ef80cb14518 Mon Sep 17 00:00:00 2001 From: Wilber Date: Wed, 9 Jun 2021 15:01:16 +0800 Subject: [PATCH 088/156] [Paddle-TRT] Add gather_nd and reduce_sum trt op. (#33324) (#33365) --- .../fluid/inference/api/analysis_predictor.cc | 2 + .../inference/tensorrt/convert/CMakeLists.txt | 2 + .../tensorrt/convert/emb_eltwise_layernorm.cc | 17 +- .../tensorrt/convert/gather_nd_op.cc | 58 +++++ .../inference/tensorrt/convert/reduce_op.cc | 90 +++++++ paddle/fluid/inference/tensorrt/op_teller.cc | 41 ++++ .../inference/tensorrt/plugin/CMakeLists.txt | 1 + .../tensorrt/plugin/gather_nd_op_plugin.cu | 229 ++++++++++++++++++ .../tensorrt/plugin/gather_nd_op_plugin.h | 132 ++++++++++ .../operators/math/bert_encoder_functor.cu | 212 ++++++++++++++-- .../ir/inference/test_trt_gather_nd_op.py | 93 +++++++ .../ir/inference/test_trt_reduce_sum_op.py | 82 +++++++ 12 files changed, 933 insertions(+), 26 deletions(-) create mode 100644 paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc create mode 100644 paddle/fluid/inference/tensorrt/convert/reduce_op.cc create mode 100644 paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.cu create mode 100644 paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.h create mode 100644 python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py create mode 100644 python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index ba729fe0492e9..baff7a6f57c52 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -1234,6 +1234,8 @@ USE_TRT_CONVERTER(roi_align); USE_TRT_CONVERTER(affine_channel); USE_TRT_CONVERTER(multiclass_nms); USE_TRT_CONVERTER(nearest_interp); +USE_TRT_CONVERTER(reduce_sum); +USE_TRT_CONVERTER(gather_nd); USE_TRT_CONVERTER(reshape); #endif diff --git a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt index 99328e6076891..c356ead850878 100644 --- a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt +++ b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt @@ -12,6 +12,8 @@ nv_library(tensorrt_converter affine_channel_op.cc multiclass_nms_op.cc nearest_interp_op.cc + reduce_op.cc + gather_nd_op.cc reshape_op.cc DEPS tensorrt_engine tensorrt_plugin operator scope framework_proto op_registry) diff --git a/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc index 66a682db07b91..04c51202f022f 100644 --- a/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc @@ -40,10 +40,19 @@ class EmbEltwiseLayerNormOpConverter : public OpConverter { auto word_emb_name = op_desc.Input("WordEmbedding").front(); auto pos_emb_name = op_desc.Input("PosEmbedding").front(); auto sent_emb_name = op_desc.Input("SentEmbedding").front(); - std::vector id_names = {word_id_name, pos_id_name, - sent_id_name}; - std::vector emb_names = {word_emb_name, pos_emb_name, - sent_emb_name}; + + std::vector id_names; + std::vector emb_names; + + if (engine_->use_oss()) { + id_names = + std::vector{word_id_name, pos_id_name, sent_id_name}; + emb_names = + std::vector{word_emb_name, pos_emb_name, sent_emb_name}; + } else { + id_names = op_desc.Input("Ids"); + emb_names = op_desc.Input("Embs"); + } int input_num = id_names.size(); diff --git a/paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc b/paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc new file mode 100644 index 0000000000000..489fc987dfec2 --- /dev/null +++ b/paddle/fluid/inference/tensorrt/convert/gather_nd_op.cc @@ -0,0 +1,58 @@ +/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/inference/tensorrt/convert/op_converter.h" +#include "paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.h" + +namespace paddle { +namespace inference { +namespace tensorrt { + +class GatherNdOpConverter : public OpConverter { + public: + void operator()(const framework::proto::OpDesc& op, + const framework::Scope& scope, bool test_mode) override { + VLOG(4) << "convert a paddle gather_nd op to tensorrt gather_nd plugin"; + framework::OpDesc op_desc(op, nullptr); + + // Declare inputs + std::vector inputs; + auto* input = engine_->GetITensor(op_desc.Input("X")[0]); + auto* index = engine_->GetITensor(op_desc.Input("Index")[0]); + inputs.emplace_back(input); + inputs.emplace_back(index); + + nvinfer1::ILayer* layer = nullptr; + bool with_fp16 = engine_->WithFp16() && !engine_->disable_trt_plugin_fp16(); + plugin::GatherNdPluginDynamic* plugin = + new plugin::GatherNdPluginDynamic(with_fp16); + layer = engine_->AddDynamicPlugin(inputs.data(), inputs.size(), plugin); + + std::string layer_name = "gather_nd (Output: "; + auto output_name = op_desc.Output("Out")[0]; + layer->getOutput(0)->setName(output_name.c_str()); + engine_->SetITensor(output_name, layer->getOutput(0)); + layer_name += output_name; + if (test_mode) { + engine_->DeclareOutput(output_name); + } + layer->setName((layer_name + ")").c_str()); + } +}; + +} // namespace tensorrt +} // namespace inference +} // namespace paddle + +REGISTER_TRT_OP_CONVERTER(gather_nd, GatherNdOpConverter); diff --git a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc new file mode 100644 index 0000000000000..66d2680fe9969 --- /dev/null +++ b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc @@ -0,0 +1,90 @@ +/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include + +#include +#include +#include + +#include "paddle/fluid/inference/tensorrt/convert/op_converter.h" + +namespace paddle { +namespace framework { +class Scope; + +namespace proto { +class OpDesc; +} // namespace proto +} // namespace framework +} // namespace paddle + +namespace paddle { +namespace inference { +namespace tensorrt { + +class ReduceSumOpConverter : public OpConverter { + public: + void operator()(const framework::proto::OpDesc& op, + const framework::Scope& scope, bool test_mode) override { + VLOG(4) << "convert a paddle reduce_sum op to tensorrt reduce layer"; + framework::OpDesc op_desc(op, nullptr); + + auto* x = engine_->GetITensor(op_desc.Input("X").front()); + nvinfer1::Dims input_shape = x->getDimensions(); + int input_dims = input_shape.nbDims; + + bool keep_dim = BOOST_GET_CONST(bool, op_desc.GetAttr("keep_dim")); + std::vector dim = + BOOST_GET_CONST(std::vector, op_desc.GetAttr("dim")); + bool reduce_all = BOOST_GET_CONST(bool, op_desc.GetAttr("reduce_all")); + + // Now we only support dynamic_shape mode. + nvinfer1::IReduceLayer* layer = nullptr; + if (reduce_all) { + uint32_t reduce_dim = 0; + for (int i = 0; i < input_dims; ++i) { + reduce_dim |= 1 << i; + } + layer = TRT_ENGINE_ADD_LAYER(engine_, Reduce, *x, + nvinfer1::ReduceOperation::kSUM, reduce_dim, + keep_dim); + } else { + auto CvtToBitMask = [&](const std::vector& dims) -> uint32_t { + uint32_t res = 0; + for (auto x : dims) { + if (x < 0) { + res |= 1 << (x + input_dims); + } else { + res |= 1 << x; + } + } + return res; + }; + layer = TRT_ENGINE_ADD_LAYER(engine_, Reduce, *x, + nvinfer1::ReduceOperation::kSUM, + CvtToBitMask(dim), keep_dim); + } + + auto output_name = op_desc.Output("Out")[0]; + RreplenishLayerAndOutput(layer, "reduce_sum", {output_name}, test_mode); + } +}; + +} // namespace tensorrt +} // namespace inference +} // namespace paddle + +REGISTER_TRT_OP_CONVERTER(reduce_sum, ReduceSumOpConverter); diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index e7a48013b07f4..07dc1a0684e8e 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "paddle/fluid/inference/tensorrt/op_teller.h" + #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/data_layout.h" @@ -122,11 +123,13 @@ struct SimpleOpTypeSetTeller : public Teller { "flatten2", "flatten", "gather", + "gather_nd", "yolo_box", "roi_align", "affine_channel", "nearest_interp", "anchor_generator", + "reduce_sum", }; }; @@ -324,6 +327,30 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, if (!with_dynamic_shape || desc.Input("Axis").size() > 0) return false; } + if (op_type == "gather_nd") { + auto* block = desc.Block(); + auto x_var_name = desc.Input("X")[0]; + auto index_var_name = desc.Input("Index")[0]; + auto* x_var_desc = block->FindVar(x_var_name); + auto* index_var_desc = block->FindVar(index_var_name); + + // The index input must be int32 datatype. + if (index_var_desc->GetDataType() != + paddle::framework::proto::VarType_Type::VarType_Type_INT32) { + VLOG(3) << "gather_nd op Index input data type must be int32"; + return false; + } + + const auto index_shape = index_var_desc->GetShape(); + const auto x_shape = x_var_desc->GetShape(); + if (x_shape.size() != index_shape.size()) { + VLOG(3) << "gather_nd op Index input dims size [" << index_shape.size() + << " ] not equal to x dims size [" << x_shape.size() << "]"; + return false; + } + if (!with_dynamic_shape) return false; + } + if (op_type == "yolo_box") { if (with_dynamic_shape) return false; bool has_attrs = @@ -658,6 +685,20 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } } + if (op_type == "reduce_sum") { + if (!with_dynamic_shape) { + VLOG(3) << "the reduce_sum does not support static shape yet"; + return false; + } + + if (!(desc.HasAttr("keep_dim") && desc.HasAttr("dim") && + desc.HasAttr("reduce_all"))) { + VLOG(3) << "the reduce_sum does not have attr (keep_dim or dim or " + "reduce_all)"; + return false; + } + } + if (op_type == "reshape" || op_type == "reshape2") { if (!desc.HasAttr("shape") || with_dynamic_shape) { return false; diff --git a/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt b/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt index 1804e6c5571d3..26125d21ad7d1 100644 --- a/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt +++ b/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt @@ -8,6 +8,7 @@ nv_library(tensorrt_plugin anchor_generator_op_plugin.cu yolo_box_op_plugin.cu roi_align_op_plugin.cu + gather_nd_op_plugin.cu DEPS enforce tensorrt_engine prelu tensor bert_encoder_functor) nv_test(test_split_plugin SRCS test_split_plugin.cc DEPS diff --git a/paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.cu new file mode 100644 index 0000000000000..5f4ac054c95b3 --- /dev/null +++ b/paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.cu @@ -0,0 +1,229 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include + +#include "NvInferRuntimeCommon.h" +#include "paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.h" +#include "paddle/fluid/inference/tensorrt/plugin/trt_plugin_factory.h" +#include "paddle/fluid/platform/place.h" + +namespace paddle { +namespace inference { +namespace tensorrt { +namespace plugin { + +#if IS_TRT_VERSION_GE(6000) + +template +__global__ void GatherNdCUDAKernel(const T* input, const int32_t* input_dims, + const IndexT* indices, T* output, + int32_t remain_size, int32_t slice_size, + int32_t end_size) { + CUDA_KERNEL_LOOP(i, remain_size * slice_size) { + int indices_i = i / slice_size; + int slice_i = i - indices_i * slice_size; // offset inside the slice + IndexT gather_i = 0; + int32_t temp = slice_size; + for (int32_t j = end_size - 1; j >= 0; --j) { + auto index_value = indices[indices_i * end_size + j]; + PADDLE_ENFORCE( + index_value >= 0 && index_value < input_dims[j], + "The index is out of bounds, " + "please check whether the dimensions of index and " + "input meet the requirements. It should " + "be less than [%d] and greater or equal to 0, but received [%d]", + input_dims[j], index_value); + gather_i += (index_value * temp); + temp *= input_dims[j]; + } + IndexT input_i = gather_i + slice_i; + *(output + i) = *(input + input_i); + } +} + +int GatherNdPluginDynamic::initialize() { return 0; } + +size_t GatherNdPluginDynamic::getSerializationSize() const { + return SerializedSize(with_fp16_); +} + +void GatherNdPluginDynamic::serialize(void* buffer) const { + SerializeValue(&buffer, with_fp16_); +} + +nvinfer1::DimsExprs GatherNdPluginDynamic::getOutputDimensions( + int output_index, const nvinfer1::DimsExprs* inputs, int nb_inputs, + nvinfer1::IExprBuilder& expr_builder) { + PADDLE_ENFORCE_EQ( + nb_inputs, 2, + platform::errors::InvalidArgument( + "The gather_nd plugin should have 2 input, but got %d.", nb_inputs)); + PADDLE_ENFORCE_EQ(output_index, 0, + platform::errors::InvalidArgument( + "When GetOutputDimensions in gather_nd " + "plugin, the output_index should be 0.")); + + nvinfer1::DimsExprs x_dims = inputs[0]; + nvinfer1::DimsExprs index_dims = inputs[1]; + + int32_t x_dims_size = x_dims.nbDims; + int32_t index_dims_size = index_dims.nbDims; + + // TODO(wilber): The result dims shoule be Index.shape[:-1] + + // X.shape[Index.shape[-1]:], but the trt DimsExprs is an expression we can't + // get the actual value. So we only support one scenario: input_dims.size == + // index_dims.size. + nvinfer1::DimsExprs ret(x_dims); + for (int i = 0; i < index_dims_size - 1; ++i) { + ret.d[i] = index_dims.d[i]; + } + + return ret; +} + +bool GatherNdPluginDynamic::supportsFormatCombination( + int pos, const nvinfer1::PluginTensorDesc* in_out, int nb_inputs, + int nb_outputs) { + PADDLE_ENFORCE_NOT_NULL( + in_out, platform::errors::InvalidArgument( + "The input of gather_nd plugin should not be nullptr.")); + + PADDLE_ENFORCE_LT( + pos, nb_inputs + nb_outputs, + platform::errors::InvalidArgument("The pos(%d) should be less than the " + "num(%d) of the input and the output.", + pos, nb_inputs + nb_outputs)); + (in_out && pos < (nb_inputs + nb_outputs)); + + const nvinfer1::PluginTensorDesc& in = in_out[pos]; + if (pos == 0) { + if (with_fp16_) { + return (in.type == nvinfer1::DataType::kFLOAT || + in.type == nvinfer1::DataType::kHALF) && + (in.format == nvinfer1::TensorFormat::kLINEAR); + } else { + return (in.type == nvinfer1::DataType::kFLOAT) && + (in.format == nvinfer1::TensorFormat::kLINEAR); + } + } else if (pos == 1) { + return in.type == nvinfer1::DataType::kINT32 && + in.format == nvinfer1::TensorFormat::kLINEAR; + } else if (pos == 2) { + return in.type == in_out[0].type && + in.format == nvinfer1::TensorFormat::kLINEAR; + } + + return true; +} + +nvinfer1::DataType GatherNdPluginDynamic::getOutputDataType( + int index, const nvinfer1::DataType* input_types, int nb_inputs) const { + return input_types[0]; +} + +int GatherNdPluginDynamic::enqueue( + const nvinfer1::PluginTensorDesc* input_desc, + const nvinfer1::PluginTensorDesc* output_desc, const void* const* inputs, + void* const* outputs, void* workspace, cudaStream_t stream) { + auto input_dims = input_desc[0].dims; + auto index_dims = input_desc[1].dims; + auto input_dims_size = input_dims.nbDims; + auto index_dims_size = index_dims.nbDims; + + std::vector input_shape, index_shape, out_shape; + for (int i = 0; i < input_dims.nbDims; i++) + input_shape.push_back(input_dims.d[i]); + for (int i = 0; i < index_dims.nbDims; i++) + index_shape.push_back(index_dims.d[i]); + // The out_shape is + // Index.shape[:-1] + X.shape[Index.shape[-1]:] + for (int i = 0; i < index_dims_size - 1; ++i) { + out_shape.emplace_back(index_shape[i]); + } + for (int i = index_shape[index_dims_size - 1]; i < input_dims_size; ++i) { + out_shape.emplace_back(input_shape[i]); + } + + // final dim + int end_size = index_shape[index_dims_size - 1]; + // remain dim + std::vector remain_ddim(index_shape.begin(), index_shape.end() - 1); + int remain_numel = std::accumulate(remain_ddim.begin(), remain_ddim.end(), 1, + std::multiplies()); + // slice size + int slice_size = 1; + for (int i = end_size; i < input_dims_size; ++i) { + slice_size *= input_shape[i]; + } + + auto input_type = input_desc[0].type; + if (input_type == nvinfer1::DataType::kFLOAT) { + VLOG(1) << "TRT Plugin DataType selected. gather_nd-->fp32"; + + const float* p_input = static_cast(inputs[0]); + const int32_t* p_index = static_cast(inputs[1]); + float* p_output = static_cast(outputs[0]); + + if (input_dims_data_ == nullptr) { + cudaMalloc(&input_dims_data_, input_shape.size() * sizeof(int)); + } + cudaMemcpyAsync(input_dims_data_, input_shape.data(), + sizeof(int) * input_shape.size(), cudaMemcpyHostToDevice, + stream); + + int block = 512; + int n = slice_size * remain_numel; + int grid = (n + block - 1) / block; + + GatherNdCUDAKernel<<>>( + p_input, input_dims_data_, p_index, p_output, remain_numel, slice_size, + end_size); + } else if (input_type == nvinfer1::DataType::kHALF) { + VLOG(1) << "TRT Plugin DataType selected. gather_nd-->fp16"; + + const half* p_input = static_cast(inputs[0]); + const int32_t* p_index = static_cast(inputs[1]); + half* p_output = static_cast(outputs[0]); + + if (input_dims_data_ == nullptr) { + cudaMalloc(&input_dims_data_, input_shape.size() * sizeof(int)); + } + cudaMemcpyAsync(input_dims_data_, input_shape.data(), + sizeof(int) * input_shape.size(), cudaMemcpyHostToDevice, + stream); + + int block = 512; + int n = slice_size * remain_numel; + int grid = (n + block - 1) / block; + + GatherNdCUDAKernel<<>>( + p_input, input_dims_data_, p_index, p_output, remain_numel, slice_size, + end_size); + } + + return cudaGetLastError() != cudaSuccess; +} +#endif + +} // namespace plugin +} // namespace tensorrt +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.h b/paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.h new file mode 100644 index 0000000000000..0a242238c81fb --- /dev/null +++ b/paddle/fluid/inference/tensorrt/plugin/gather_nd_op_plugin.h @@ -0,0 +1,132 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include "paddle/fluid/framework/tensor.h" +#include "paddle/fluid/inference/tensorrt/plugin/trt_plugin.h" + +namespace paddle { +namespace inference { +namespace tensorrt { +namespace plugin { + +#if IS_TRT_VERSION_GE(6000) +class GatherNdPluginDynamic : public DynamicPluginTensorRT { + public: + explicit GatherNdPluginDynamic(bool with_fp16) { with_fp16_ = with_fp16; } + + GatherNdPluginDynamic(void const* serial_data, size_t serial_length) { + DeserializeValue(&serial_data, &serial_length, &with_fp16_); + } + + nvinfer1::IPluginV2DynamicExt* clone() const override { + return new GatherNdPluginDynamic(with_fp16_); + } + + const char* getPluginType() const override { return "gather_nd_plugin"; } + int getNbOutputs() const override { return 1; } + int initialize() override; + + size_t getSerializationSize() const override; + void serialize(void* buffer) const override; + + nvinfer1::DimsExprs getOutputDimensions( + int outputIndex, const nvinfer1::DimsExprs* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) override; + + bool supportsFormatCombination(int pos, + const nvinfer1::PluginTensorDesc* inOut, + int nbInputs, int nbOutputs) override; + + void configurePlugin(const nvinfer1::DynamicPluginTensorDesc* in, + int nbInputs, + const nvinfer1::DynamicPluginTensorDesc* out, + int nbOutputs) override {} + + size_t getWorkspaceSize(const nvinfer1::PluginTensorDesc* inputs, + int nbInputs, + const nvinfer1::PluginTensorDesc* outputs, + int nbOutputs) const override { + return 0; + } + + int enqueue(const nvinfer1::PluginTensorDesc* inputDesc, + const nvinfer1::PluginTensorDesc* outputDesc, + const void* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) override; + nvinfer1::DataType getOutputDataType(int index, + const nvinfer1::DataType* inputTypes, + int nbInputs) const override; + + void destroy() override { + if (input_dims_data_) { + cudaFree(input_dims_data_); + } + delete this; + } + + private: + int32_t* input_dims_data_{nullptr}; +}; + +class GatherNdPluginDynamicCreator : public nvinfer1::IPluginCreator { + public: + GatherNdPluginDynamicCreator() {} + const char* getPluginName() const override { return "gather_nd_plugin"; } + + const char* getPluginVersion() const override { return "1"; } + + const nvinfer1::PluginFieldCollection* getFieldNames() override { + return &field_collection_; + } + + nvinfer1::IPluginV2* createPlugin( + const char* name, const nvinfer1::PluginFieldCollection* fc) override { + return nullptr; + } + + nvinfer1::IPluginV2* deserializePlugin(const char* name, + const void* serial_data, + size_t serial_length) override { + auto plugin = new GatherNdPluginDynamic(serial_data, serial_length); + return plugin; + } + + void setPluginNamespace(const char* lib_namespace) override { + plugin_namespace_ = lib_namespace; + } + + const char* getPluginNamespace() const override { + return plugin_namespace_.c_str(); + } + + private: + std::string plugin_namespace_; + std::string plugin_name_; + nvinfer1::PluginFieldCollection field_collection_{0, nullptr}; + std::vector plugin_attributes_; +}; + +REGISTER_TRT_PLUGIN_V2(GatherNdPluginDynamicCreator); +#endif + +} // namespace plugin +} // namespace tensorrt +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/operators/math/bert_encoder_functor.cu b/paddle/fluid/operators/math/bert_encoder_functor.cu index 512f9c62415e5..4d7218cd89e04 100644 --- a/paddle/fluid/operators/math/bert_encoder_functor.cu +++ b/paddle/fluid/operators/math/bert_encoder_functor.cu @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include + #include "paddle/fluid/framework/tensor.h" #include "paddle/fluid/framework/tensor_util.h" #include "paddle/fluid/operators/math/bert_encoder_functor.h" @@ -311,6 +312,156 @@ __global__ void SoftmaxKernelWithEltadd2( #endif } +template +__global__ void SoftmaxKernelWithEltaddForLarge(T *qk_buf, const T *bias_qk, + const int batch_size, + const int head_num, + const int seq_len, + const unsigned mask) { + int qk_offset = blockIdx.x * seq_len; + assert(blockDim.x % 32 == 0); + + T stride_max = -1e20f; + for (int i = 0; i < seq_len; i += blockDim.x) { + stride_max = qk_buf[threadIdx.x + i + qk_offset] + + bias_qk[threadIdx.x + i + qk_offset] > + stride_max + ? qk_buf[threadIdx.x + i + qk_offset] + + bias_qk[threadIdx.x + i + qk_offset] + : stride_max; + } + T max_val = blockReduceMax(stride_max, mask); + + T stride_sum = 0.f; + for (int i = 0; i < seq_len; i += blockDim.x) { + stride_sum += __expf(qk_buf[threadIdx.x + i + qk_offset] + + bias_qk[threadIdx.x + i + qk_offset] - max_val); + } + T sum_val = blockReduceSum(stride_sum, mask); + + for (int i = 0; i < seq_len; i += blockDim.x) { + qk_buf[threadIdx.x + i + qk_offset] = + (T)(__expf(qk_buf[threadIdx.x + i + qk_offset] + + bias_qk[threadIdx.x + i + qk_offset] - max_val) / + sum_val); + } +} + +// HIP defined __HIP_NO_HALF_CONVERSIONS__ +#ifndef __HIPCC__ // @{ Half kernel: SoftmaxKernelWithEltadd +template <> +__global__ void SoftmaxKernelWithEltaddForLarge( + half *qk_buf, const half *bias_qk, const int batch_size, const int head_num, + const int seq_len, const unsigned mask) { +#if CUDA_ARCH_FP16_SUPPORTED(__CUDA_ARCH__) + int qk_offset = blockIdx.x * seq_len; + assert(blockDim.x % 32 == 0); + + float stride_max = -1e20f; + for (int i = 0; i < seq_len; i += blockDim.x) { + float tmp = static_cast(qk_buf[threadIdx.x + i + qk_offset] + + bias_qk[threadIdx.x + i + qk_offset]); + stride_max = tmp > stride_max ? tmp : stride_max; + } + float max_val = blockReduceMax(stride_max, mask); + + float stride_sum = 0.f; + for (int i = 0; i < seq_len; i += blockDim.x) { + float tmp = static_cast(qk_buf[threadIdx.x + i + qk_offset] + + bias_qk[threadIdx.x + i + qk_offset]); + stride_sum += __expf(tmp - max_val); + } + float sum_val = blockReduceSum(stride_sum, mask); + + for (int i = 0; i < seq_len; i += blockDim.x) { + float tmp = + __expf(static_cast(qk_buf[threadIdx.x + i + qk_offset] + + bias_qk[threadIdx.x + i + qk_offset]) - + max_val); + qk_buf[threadIdx.x + i + qk_offset] = (half)(tmp / sum_val); + } +#endif +} +#endif // @} End Half kernel: SoftmaxKernelWithEltadd + +template +__global__ void SoftmaxKernelWithEltaddForLarge2(T *qk_buf_, const T *bias_qk_, + const int batch_size, + const int head_num, + const int seq_len, + const unsigned mask) { + int qk_offset = blockIdx.x * seq_len; + assert(blockDim.x % 32 == 0); + + float2 stride_max = make_float2(-1e20f, -1e20f); + for (int i = 0; i < seq_len; i += blockDim.x) { + float2 cur = ToFloat2(qk_buf_[threadIdx.x + i + qk_offset] + + bias_qk_[threadIdx.x + i + qk_offset]); + stride_max.x = max(stride_max.x, cur.x); + stride_max.y = max(stride_max.y, cur.y); + } + float max_val = blockReduceMax(max(stride_max.x, stride_max.y), mask); + + float2 stride_sum = make_float2(0.f, 0.f); + for (int i = 0; i < seq_len; i += blockDim.x) { + float2 cur = ToFloat2(qk_buf_[threadIdx.x + i + qk_offset] + + bias_qk_[threadIdx.x + i + qk_offset]); + stride_sum.x += __expf(cur.x - max_val); + stride_sum.y += __expf(cur.y - max_val); + } + + float sum_val = + blockReduceSum(stride_sum.x + stride_sum.y, mask) + 1e-6f; + + for (int i = 0; i < seq_len; i += blockDim.x) { + float2 cur = ToFloat2(qk_buf_[threadIdx.x + i + qk_offset] + + bias_qk_[threadIdx.x + i + qk_offset]); + qk_buf_[threadIdx.x + i + qk_offset] = FloatsToPair( + __expf(cur.x - max_val) / sum_val, __expf(cur.y - max_val) / sum_val); + } +} + +template <> +__global__ void SoftmaxKernelWithEltaddForLarge2( + half2 *qk_buf_, const half2 *bias_qk_, const int batch_size, + const int head_num, const int seq_len, const unsigned mask) { +// operator "+" of half only suppotted after cuda version 10.0 +// HIP defined __HIP_NO_HALF_CONVERSIONS__ in hip.cmake +#if defined(PADDLE_WITH_CUDA) && \ + (CUDA_ARCH_FP16_SUPPORTED(__CUDA_ARCH__) && CUDA_VERSION >= 10000) + + int qk_offset = blockIdx.x * seq_len; + assert(blockDim.x % 32 == 0); + + float2 stride_max = make_float2(-1e20f, -1e20f); + for (int i = 0; i < seq_len; i += blockDim.x) { + float2 cur = ToFloat2(qk_buf_[threadIdx.x + i + qk_offset] + + bias_qk_[threadIdx.x + i + qk_offset]); + stride_max.x = max(stride_max.x, cur.x); + stride_max.y = max(stride_max.y, cur.y); + } + float max_val = blockReduceMax(max(stride_max.x, stride_max.y), mask); + + float2 stride_sum = make_float2(0.f, 0.f); + for (int i = 0; i < seq_len; i += blockDim.x) { + float2 cur = ToFloat2(qk_buf_[threadIdx.x + i + qk_offset] + + bias_qk_[threadIdx.x + i + qk_offset]); + stride_sum.x += __expf(cur.x - max_val); + stride_sum.y += __expf(cur.y - max_val); + } + + float sum_val = + blockReduceSum(stride_sum.x + stride_sum.y, mask) + 1e-6f; + + for (int i = 0; i < seq_len; i += blockDim.x) { + float2 cur = ToFloat2(qk_buf_[threadIdx.x + i + qk_offset] + + bias_qk_[threadIdx.x + i + qk_offset]); + qk_buf_[threadIdx.x + i + qk_offset] = FloatsToPair( + __expf(cur.x - max_val) / sum_val, __expf(cur.y - max_val) / sum_val); + } +#endif +} + template inline void MatMulWithHeadQK(const platform::CUDADeviceContext &context, int head_num, int seq_len, int size_per_head, @@ -332,31 +483,48 @@ inline void MatMulWithHeadQK(const platform::CUDADeviceContext &context, reinterpret_cast(qk_buf_), batch_size * head_num, seq_len * size_per_head, seq_len * size_per_head); - int grid = batch_size * head_num * seq_len; - int block = seq_len; - - // Align block to 32, also limit seq_len to max block size. - PADDLE_ENFORCE_LE(seq_len, 1024, platform::errors::InvalidArgument( - "seq_len should <= 1024, " - "but received seq_len is:%d", - seq_len)); - if (seq_len % 2 == 0) { - block = (seq_len <= 64) ? 32 : ((seq_len + 63) / 64) * 32; - if (std::is_same::value) { - SoftmaxKernelWithEltadd2<<>>( - reinterpret_cast(qk_buf_), - reinterpret_cast(bias_qk), batch_size, head_num, - seq_len / 2, FINAL_MASK); + if (seq_len <= 1024) { + int grid = batch_size * head_num * seq_len; + int block = seq_len; + + // Align block to 32, also limit seq_len to max block size. + if (seq_len % 2 == 0) { + block = (seq_len <= 64) ? 32 : ((seq_len + 63) / 64) * 32; + if (std::is_same::value) { + SoftmaxKernelWithEltadd2<<>>( + reinterpret_cast(qk_buf_), + reinterpret_cast(bias_qk), batch_size, head_num, + seq_len / 2, FINAL_MASK); + } else { + SoftmaxKernelWithEltadd2<__half2><<>>( + reinterpret_cast<__half2 *>(qk_buf_), + reinterpret_cast(bias_qk), batch_size, head_num, + seq_len / 2, FINAL_MASK); + } } else { - SoftmaxKernelWithEltadd2<__half2><<>>( - reinterpret_cast<__half2 *>(qk_buf_), - reinterpret_cast(bias_qk), batch_size, head_num, - seq_len / 2, FINAL_MASK); + block = (seq_len <= 32) ? 32 : ((seq_len + 31) / 32) * 32; + SoftmaxKernelWithEltadd<<>>( + qk_buf_, bias_qk, batch_size, head_num, seq_len, FINAL_MASK); } } else { - block = (seq_len <= 32) ? 32 : ((seq_len + 31) / 32) * 32; - SoftmaxKernelWithEltadd<<>>( - qk_buf_, bias_qk, batch_size, head_num, seq_len, FINAL_MASK); + int grid = batch_size * head_num * seq_len; + int block = 512; + if (seq_len % 2 == 0) { + if (std::is_same::value) { + SoftmaxKernelWithEltaddForLarge2<<>>( + reinterpret_cast(qk_buf_), + reinterpret_cast(bias_qk), batch_size, head_num, + seq_len / 2, FINAL_MASK); + } else { + SoftmaxKernelWithEltaddForLarge2<__half2><<>>( + reinterpret_cast<__half2 *>(qk_buf_), + reinterpret_cast(bias_qk), batch_size, head_num, + seq_len / 2, FINAL_MASK); + } + } else { + SoftmaxKernelWithEltaddForLarge<<>>( + qk_buf_, bias_qk, batch_size, head_num, seq_len, FINAL_MASK); + } } } diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py new file mode 100644 index 0000000000000..75f5328ac1c41 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py @@ -0,0 +1,93 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +from inference_pass_test import InferencePassTest +import paddle.fluid as fluid +import paddle.fluid.core as core +from paddle.fluid.core import PassVersionChecker +from paddle.fluid.core import AnalysisConfig + + +class TRTGatherNdTest(InferencePassTest): + def setUp(self): + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data(name="data", shape=[-1, 3, 4], dtype="float32") + index = fluid.data(name="index", shape=[-1, 2, 2], dtype="int32") + gather_nd = fluid.layers.gather_nd(data, index) + out = fluid.layers.batch_norm(gather_nd, is_test=True) + + self.feeds = { + "data": np.random.random([2, 3, 4]).astype("float32"), + "index": + np.array([[[0, 1], [1, 0]], [[1, 2], [0, 1]]]).astype("int32"), + } + self.enable_trt = True + self.trt_parameters = TRTGatherNdTest.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + self.fetch_list = [out] + self.dynamic_shape_params = TRTGatherNdTest.DynamicShapeParam({ + 'data': [1, 3, 4], + 'index': [1, 2, 2] + }, {'data': [3, 3, 4], + 'index': [3, 2, 2]}, {'data': [3, 3, 4], + 'index': [3, 2, 2]}, False) + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu, flatten=True) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + +class TRTGatherNdFp16Test(InferencePassTest): + def setUp(self): + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name="data", shape=[-1, 5120, 768], dtype="float32") + index = fluid.data(name="index", shape=[-1, 4096, 2], dtype="int32") + gather_nd = fluid.layers.gather_nd(data, index) + out = fluid.layers.batch_norm(gather_nd, is_test=True) + + index_data = np.zeros((1, 4096, 2), dtype='int32') + self.feeds = { + "data": np.random.random([1, 5120, 768]).astype("float32"), + "index": index_data, + } + self.enable_trt = True + self.trt_parameters = TRTGatherNdFp16Test.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + self.fetch_list = [out] + self.dynamic_shape_params = TRTGatherNdFp16Test.DynamicShapeParam({ + 'data': [1, 5120, 768], + 'index': [1, 4096, 2] + }, {'data': [3, 5120, 768], + 'index': + [3, 4096, 2]}, {'data': [3, 5120, 768], + 'index': [3, 4096, 2]}, False) + + def test_check_output(self, atol=1e-3): + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu, flatten=True) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py new file mode 100644 index 0000000000000..bb5e8e99b0926 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py @@ -0,0 +1,82 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +from inference_pass_test import InferencePassTest +import paddle.fluid as fluid +import paddle.fluid.core as core +from paddle.fluid.core import PassVersionChecker +from paddle.fluid.core import AnalysisConfig + + +class TRTReduceSumTest(InferencePassTest): + def setUp(self): + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name="data", shape=[-1, 3, 10, 768], dtype="float32") + reduce_sum = fluid.layers.reduce_sum( + data, dim=[2, -1], keep_dim=True) + out = fluid.layers.batch_norm(reduce_sum, is_test=True) + + self.feeds = { + "data": np.random.random([3, 3, 10, 768]).astype("float32"), + } + self.enable_trt = True + self.trt_parameters = TRTReduceSumTest.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + self.fetch_list = [out] + self.dynamic_shape_params = TRTReduceSumTest.DynamicShapeParam({ + 'data': [1, 3, 8, 8] + }, {'data': [3, 3, 10, 768]}, {'data': [3, 3, 10, 768]}, False) + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu, flatten=True) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + +class TRTReduceSumAllTest(InferencePassTest): + def setUp(self): + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name="data", shape=[-1, 3, 10, 768], dtype="float32") + reduce_sum = fluid.layers.reduce_sum(data, keep_dim=True) + out = fluid.layers.batch_norm(reduce_sum, is_test=True) + + self.feeds = { + "data": np.random.random([3, 3, 10, 768]).astype("float32"), + } + self.enable_trt = True + self.trt_parameters = TRTReduceSumAllTest.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + self.fetch_list = [out] + self.dynamic_shape_params = TRTReduceSumAllTest.DynamicShapeParam({ + 'data': [1, 3, 8, 8] + }, {'data': [3, 3, 10, 768]}, {'data': [3, 3, 10, 768]}, False) + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu, flatten=True) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + +if __name__ == "__main__": + unittest.main() From d496722466cc48d6ed1ce6f49d9ecbe52d94c791 Mon Sep 17 00:00:00 2001 From: feng_shuai Date: Wed, 9 Jun 2021 20:26:13 +0800 Subject: [PATCH 089/156] fix the bug of yolo_box which can't run on nano and tx2 (#33422) (#33442) --- paddle/fluid/operators/detection/yolo_box_op.cu | 9 ++++++++- paddle/fluid/platform/gpu_launch_config.h | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/operators/detection/yolo_box_op.cu b/paddle/fluid/operators/detection/yolo_box_op.cu index 65dc73ef38323..c8b36ad606fdd 100644 --- a/paddle/fluid/operators/detection/yolo_box_op.cu +++ b/paddle/fluid/operators/detection/yolo_box_op.cu @@ -111,7 +111,14 @@ class YoloBoxOpCUDAKernel : public framework::OpKernel { platform::GpuLaunchConfig config = platform::GetGpuLaunchConfig1D(ctx.cuda_device_context(), n * box_num); - KeYoloBoxFw<<<<>>( input_data, imgsize_data, boxes_data, scores_data, conf_thresh, anchors_data, n, h, w, an_num, class_num, box_num, input_size_h, diff --git a/paddle/fluid/platform/gpu_launch_config.h b/paddle/fluid/platform/gpu_launch_config.h index 6c265677d63e9..4da91b4e764a5 100644 --- a/paddle/fluid/platform/gpu_launch_config.h +++ b/paddle/fluid/platform/gpu_launch_config.h @@ -37,6 +37,7 @@ struct GpuLaunchConfig { dim3 theory_thread_count = dim3(1, 1, 1); dim3 thread_per_block = dim3(1, 1, 1); dim3 block_per_grid = dim3(1, 1, 1); + int compute_capability = 0; }; inline GpuLaunchConfig GetGpuLaunchConfig1D( @@ -67,11 +68,14 @@ inline GpuLaunchConfig GetGpuLaunchConfig1D( std::min(max_threads, context.GetMaxThreadsPerBlock()); const int block_count = std::min(DivUp(physical_thread_count, thread_per_block), sm); + // Get compute_capability + const int capability = context.GetComputeCapability(); GpuLaunchConfig config; config.theory_thread_count.x = theory_thread_count; config.thread_per_block.x = thread_per_block; config.block_per_grid.x = block_count; + config.compute_capability = capability; return config; } From c4a417f5a74cf602f2af75d4a5c7a96a60e655c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=98=8E=E5=86=AC?= <78149749+winter-wang@users.noreply.github.com> Date: Thu, 10 Jun 2021 10:24:17 +0800 Subject: [PATCH 090/156] fix the bug in repeated_fc_relu_fuse_pass.test=develop (#33386) (#33431) --- .../fluid/framework/ir/repeated_fc_relu_fuse_pass.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc b/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc index bf59c14000516..4c87b63625c1f 100644 --- a/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc +++ b/paddle/fluid/framework/ir/repeated_fc_relu_fuse_pass.cc @@ -66,9 +66,13 @@ static bool IsFCWithPaddingWeights(Node* n) { } static bool IsParamOfFC(Node* n, const std::string& param_name) { - if (IsInputOfFC(n) && n->inputs.empty() && - (n->Name() == n->outputs[0]->Op()->Input(param_name)[0])) { - return true; + if (IsInputOfFC(n) && n->inputs.empty()) { + for (auto* out : n->outputs) { + if (out->Op()->Type() == "fc" && + n->Name() == out->Op()->Input(param_name)[0]) { + return true; + } + } } return false; } From 03f46685caf393e25aea19644bb7d5b406531eec Mon Sep 17 00:00:00 2001 From: wangguanzhong Date: Thu, 10 Jun 2021 10:42:19 +0800 Subject: [PATCH 091/156] fix aligned in roi_align (#33446) --- paddle/fluid/operators/roi_align_op.cu | 15 +++++++++------ paddle/fluid/operators/roi_align_op.h | 12 +++++++++--- .../fluid/tests/unittests/test_roi_align_op.py | 7 ++++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/operators/roi_align_op.cu b/paddle/fluid/operators/roi_align_op.cu index d6ba399439d02..934802f6a9e0e 100644 --- a/paddle/fluid/operators/roi_align_op.cu +++ b/paddle/fluid/operators/roi_align_op.cu @@ -124,8 +124,10 @@ __global__ void GPUROIAlignForward( T roi_width = roi_xmax - roi_xmin; T roi_height = roi_ymax - roi_ymin; - roi_width = max(roi_width, static_cast(1.)); - roi_height = max(roi_height, static_cast(1.)); + if (!continuous_coordinate) { + roi_width = max(roi_width, static_cast(1.)); + roi_height = max(roi_height, static_cast(1.)); + } T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); @@ -138,7 +140,7 @@ __global__ void GPUROIAlignForward( : ceil(roi_height / pooled_height); int roi_bin_grid_w = (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); - const T count = roi_bin_grid_h * roi_bin_grid_w; + const T count = max(roi_bin_grid_h * roi_bin_grid_w, 1); T output_val = 0; for (int iy = 0; iy < roi_bin_grid_h; iy++) { const T y = roi_ymin + ph * bin_size_h + @@ -180,9 +182,10 @@ __global__ void GPUROIAlignBackward( T roi_width = roi_xmax - roi_xmin; T roi_height = roi_ymax - roi_ymin; - roi_width = max(roi_width, static_cast(1.)); - roi_height = max(roi_height, static_cast(1.)); - + if (!continuous_coordinate) { + roi_width = max(roi_width, static_cast(1.)); + roi_height = max(roi_height, static_cast(1.)); + } T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); diff --git a/paddle/fluid/operators/roi_align_op.h b/paddle/fluid/operators/roi_align_op.h index 46564ed4f629d..29c9268d5241c 100644 --- a/paddle/fluid/operators/roi_align_op.h +++ b/paddle/fluid/operators/roi_align_op.h @@ -226,8 +226,10 @@ class CPUROIAlignOpKernel : public framework::OpKernel { T roi_width = roi_xmax - roi_xmin; T roi_height = roi_ymax - roi_ymin; - roi_width = std::max(roi_width, static_cast(1.)); - roi_height = std::max(roi_height, static_cast(1.)); + if (!aligned) { + roi_width = std::max(roi_width, static_cast(1.)); + roi_height = std::max(roi_height, static_cast(1.)); + } T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); @@ -239,7 +241,7 @@ class CPUROIAlignOpKernel : public framework::OpKernel { int roi_bin_grid_w = (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); - const T count = roi_bin_grid_h * roi_bin_grid_w; + const T count = std::max(roi_bin_grid_h * roi_bin_grid_w, 1); Tensor pre_pos; Tensor pre_w; int pre_size = count * out_stride[1]; @@ -362,6 +364,10 @@ class CPUROIAlignGradOpKernel : public framework::OpKernel { T roi_height = roi_ymax - roi_ymin; roi_width = std::max(roi_width, static_cast(1.)); roi_height = std::max(roi_height, static_cast(1.)); + if (!aligned) { + roi_width = std::max(roi_width, static_cast(1.)); + roi_height = std::max(roi_height, static_cast(1.)); + } T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); diff --git a/python/paddle/fluid/tests/unittests/test_roi_align_op.py b/python/paddle/fluid/tests/unittests/test_roi_align_op.py index 7d030855d114e..7fab4017ab0ba 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_align_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_align_op.py @@ -129,8 +129,9 @@ def calc_roi_align(self): roi_width = roi_xmax - roi_xmin roi_height = roi_ymax - roi_ymin - roi_width = max(roi_width, 1) - roi_height = max(roi_height, 1) + if not self.aligned: + roi_width = max(roi_width, 1) + roi_height = max(roi_height, 1) bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) @@ -138,7 +139,7 @@ def calc_roi_align(self): math.ceil(roi_height / self.pooled_height) roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ math.ceil(roi_width / self.pooled_width) - count = int(roi_bin_grid_h * roi_bin_grid_w) + count = max(int(roi_bin_grid_h * roi_bin_grid_w), 1) pre_size = count * self.pooled_width * self.pooled_height bilinear_pos, bilinear_w = self.pre_calc(x_i, roi_xmin, roi_ymin, int(roi_bin_grid_h), From fe841790830e6b15438c1a1011e21141f65aa80b Mon Sep 17 00:00:00 2001 From: lilong12 Date: Thu, 10 Jun 2021 16:05:45 +0800 Subject: [PATCH 092/156] fix the bug in the creation of pp groups to avoid hang (#32890) (#33473) * update, test=develop --- .../fleet/meta_optimizers/common.py | 15 +- .../meta_optimizers/pipeline_optimizer.py | 3 + python/paddle/fluid/optimizer.py | 6 +- .../unittests/pipeline_mnist_multi_device.py | 159 ++++++++++++++++++ .../fluid/tests/unittests/test_pipeline.py | 9 + 5 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/pipeline_mnist_multi_device.py diff --git a/python/paddle/distributed/fleet/meta_optimizers/common.py b/python/paddle/distributed/fleet/meta_optimizers/common.py index 707284a784c38..9e891062bcbcc 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/common.py +++ b/python/paddle/distributed/fleet/meta_optimizers/common.py @@ -77,9 +77,12 @@ def _init_communicator(self, wait_port, global_ring_id=None, sync=True): - nranks = len(endpoints) - other_endpoints = endpoints[:] - other_endpoints.remove(current_endpoint) + # if current_endpoint is None, it means just for sync, + # no group is created. + if current_endpoint: + nranks = len(endpoints) + other_endpoints = endpoints[:] + other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: wait_server_ready(other_endpoints) @@ -117,6 +120,12 @@ def _add_sync_by_allreduce(block): attrs={OP_ROLE_KEY: OpRole.Forward}) block = program.global_block() + if current_endpoint is None: + assert endpoints is None + assert sync + _add_sync_by_allreduce(block) + return + if core.is_compiled_with_cuda(): comm_id_var = block.create_var( name=unique_name.generate('nccl_id'), diff --git a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py index a0bf4cc5bc097..481b90910def1 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py @@ -138,6 +138,9 @@ def _init_process_group(self, pipeline_pair, pipeline_ring_map): first_node = pair[0] + start_index second_node = pair[1] + start_index if self.rank != first_node and self.rank != second_node: + collective_helper._init_communicator( + self.startup_program, None, None, None, None, False, + self.global_ring_id, True) continue pipeline_endpoints = [ self.endpoints[first_node], self.endpoints[second_node] diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 41d5401074548..cf2048b38b53f 100755 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -3856,6 +3856,7 @@ def _insert_allreduce_op(self, op_idx, block): 'out_dtype': out_var.dtype, self._op_role_key: self._op_role.Optimize }) + offset += 1 return offset def _create_vars(self, block, ori_block): @@ -4364,12 +4365,15 @@ def _insert_send_recv(cur_id, prev_id): 'ring_id': ring_id }) extra_index_info['index'] += 1 + var_shape = list(var.shape) + var_shape[0] = self.micro_batch_size if var_shape[ + 0] < 0 else var_shape[0] block._insert_op_without_sync( index=index + extra_index_info['index'], type='recv_v2', outputs={'Out': [var]}, attrs={ - 'out_shape': var.shape, + 'out_shape': var_shape, 'dtype': var.dtype, self._op_device_key: cur_dev, self._op_role_key: op_role, diff --git a/python/paddle/fluid/tests/unittests/pipeline_mnist_multi_device.py b/python/paddle/fluid/tests/unittests/pipeline_mnist_multi_device.py new file mode 100644 index 0000000000000..7211bd3e92f79 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/pipeline_mnist_multi_device.py @@ -0,0 +1,159 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import numpy as np +import argparse +import time +import math + +import paddle +import paddle.fluid as fluid +import paddle.fluid.profiler as profiler +from paddle.fluid import core +import unittest +from multiprocessing import Process +import os +import signal +from functools import reduce +from test_dist_base import TestDistRunnerBase, runtime_main +import paddle.distributed.fleet as fleet + +paddle.enable_static() + +DTYPE = "float32" +paddle.dataset.mnist.fetch() + +# Fix seed for test +fluid.default_startup_program().random_seed = 1 +fluid.default_main_program().random_seed = 1 + + +def cnn_model(data): + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=data, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( + value=0.01))) + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( + value=0.01))) + + SIZE = 10 + input_shape = conv_pool_2.shape + param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] + scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + + with fluid.device_guard("gpu:1"): + predict = fluid.layers.fc( + input=conv_pool_2, + size=SIZE, + act="softmax", + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01))) + # To cover @RENAMED@GRADIENT + predict2 = fluid.layers.fc( + input=conv_pool_1, + size=SIZE, + act="softmax", + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01))) + predict += predict2 + return predict + + +class TestDistMnist2x2(TestDistRunnerBase): + def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): + # Input data + with fluid.device_guard("gpu:0"): + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype=DTYPE) + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + + if dist_strategy: + data_loader = fluid.io.DataLoader.from_generator( + feed_list=[images, label], + capacity=64, + use_double_buffer=False, + iterable=False) + # Train program + predict = cnn_model(images) + with fluid.device_guard("gpu:1"): + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(x=cost) + + # Evaluator + with fluid.device_guard("gpu:1"): + batch_size_tensor = fluid.layers.create_tensor(dtype='int64') + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor) + + inference_program = fluid.default_main_program().clone() + base_lr = self.lr + passes = [30, 60, 80, 90] + steps_per_pass = 10 + bd = [steps_per_pass * p for p in passes] + lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] + lr_val = fluid.layers.piecewise_decay(boundaries=bd, values=lr) + opt = fluid.optimizer.Momentum( + learning_rate=lr_val, + momentum=0.9, + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + + acc_steps = 2 # accumulated steps for pipeline + if dist_strategy: + # Reader + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size) + fleet.init(is_collective=True) + strategy = fleet.DistributedStrategy() + strategy.pipeline = True + strategy.amp = True + strategy.pipeline_configs = { + 'micro_batch_size': batch_size, + 'schedule_mode': 'F-then-B', + 'accumulate_steps': acc_steps + } + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy) + dist_opt.minimize(avg_cost) + else: + opt.minimize(avg_cost) + # Reader + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps) + + if dist_strategy: + return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader + else: + return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + + +if __name__ == "__main__": + runtime_main(TestDistMnist2x2) diff --git a/python/paddle/fluid/tests/unittests/test_pipeline.py b/python/paddle/fluid/tests/unittests/test_pipeline.py index cd592416c1a51..1be10113a5591 100644 --- a/python/paddle/fluid/tests/unittests/test_pipeline.py +++ b/python/paddle/fluid/tests/unittests/test_pipeline.py @@ -44,6 +44,15 @@ def test_dist_train(self): check_error_log=True, log_name=flag_name) + def test_dist_train_multi_device(self): + import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): + self.check_with_place( + "pipeline_mnist_multi_device.py", + check_error_log=True, + delta=1e0, + log_name=flag_name) + def test_dist_train_one_device(self): import paddle.fluid as fluid if fluid.core.is_compiled_with_cuda(): From 9035fd2e5564b57010b67e26b5f8153ab7cf77e6 Mon Sep 17 00:00:00 2001 From: Wenyu Date: Thu, 10 Jun 2021 16:52:53 +0800 Subject: [PATCH 093/156] [cherry-pick] Fix retry error in download when exception occurs #32816 (#33454) * fix retry in download when exception occurs * add test_retry_exception --- python/paddle/tests/test_download.py | 7 +++++++ python/paddle/utils/download.py | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python/paddle/tests/test_download.py b/python/paddle/tests/test_download.py index b8af7f6a80e72..4be2dde1bccb1 100644 --- a/python/paddle/tests/test_download.py +++ b/python/paddle/tests/test_download.py @@ -70,6 +70,13 @@ def test_get_path_from_url(self): for url in urls: get_path_from_url(url, root_dir='./test') + def test_retry_exception(self, ): + with self.assertRaises(RuntimeError): + from paddle.utils.download import _download + _download( + 'www.baidu.com', + './test', ) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index dda8abeff21c0..3ad627ddea927 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -186,7 +186,15 @@ def _download(url, path, md5sum=None): logger.info("Downloading {} from {}".format(fname, url)) - req = requests.get(url, stream=True) + try: + req = requests.get(url, stream=True) + except Exception as e: # requests.exceptions.ConnectionError + logger.info( + "Downloading {} from {} failed {} times with exception {}". + format(fname, url, retry_cnt + 1, str(e))) + time.sleep(1) + continue + if req.status_code != 200: raise RuntimeError("Downloading from {} failed with code " "{}!".format(url, req.status_code)) From 1cdf69b21519ff6d1639f6d127beab857e5dce43 Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Thu, 10 Jun 2021 17:27:04 +0800 Subject: [PATCH 094/156] [cherry pick] add random state generate in DataLoader worker (#33434) * add random state generate in DataLoader worker. test=develop * fix license and __all__. test=develop * fix unittest. test=develop --- python/paddle/fluid/dataloader/worker.py | 92 +++++++++++++++++++ .../test_multiprocess_dataloader_dataset.py | 14 +++ 2 files changed, 106 insertions(+) diff --git a/python/paddle/fluid/dataloader/worker.py b/python/paddle/fluid/dataloader/worker.py index 26bd1f06e12e8..409f55efebc8a 100644 --- a/python/paddle/fluid/dataloader/worker.py +++ b/python/paddle/fluid/dataloader/worker.py @@ -168,6 +168,89 @@ def reraise(self): raise self.exc_type(msg) +# The function `_generate_states` is adapted from `numpy.random.SeedSequence` +# from https://github.com/numpy/numpy/blob/main/numpy/random/bit_generator.pyx +# Here is the copyright: + +# SeedSequence is derived from Melissa E. O'Neill's C++11 `std::seed_seq` +# implementation, as it has a lot of nice properties that we want. +# https://gist.github.com/imneme/540829265469e673d045 +# http://www.pcg-random.org/posts/developing-a-seed_seq-alternative.html + +# The MIT License (MIT) + +# Copyright (c) 2015 Melissa E. O'Neill +# Copyright (c) 2019 NumPy Developers +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +INIT_A = 0x43b0d7e5 +MULT_A = 0x931e8875 +INIT_B = 0x8b51f9dd +MULT_B = 0x58f38ded +MIX_MULT_L = 0xca01f9dd +MIX_MULT_R = 0x4973f715 +XSHIFT = np.dtype(np.uint32).itemsize * 8 // 2 +MASK32 = 0xFFFFFFFF + + +def _generate_states(base_seed=0, worker_id=0): + # init hash constant + hash_const_A = INIT_A + hash_const_B = INIT_B + + def hash(value): + nonlocal hash_const_A + value = (value ^ hash_const_A) & MASK32 + hash_const_A = (hash_const_A * MULT_A) & MASK32 + value = (value * hash_const_A) & MASK32 + value = (value ^ (value >> XSHIFT)) & MASK32 + return value + + def mix(x, y): + result_x = (MIX_MULT_L * x) & MASK32 + result_y = (MIX_MULT_R * y) & MASK32 + result = (result_x - result_y) & MASK32 + result = (result ^ (result >> XSHIFT)) & MASK32 + return result + + # init entropys with based_seed and worker_id and calculate pool + entropys = [worker_id, base_seed & MASK32, base_seed >> 32, 0] + pool = [hash(entropy) for entropy in entropys] + + # mix all bits together + for i in range(len(pool)): + for j in range(len(pool)): + if i != j: + pool[j] = mix(pool[j], hash(pool[i])) + + states = [] + for p in pool: + state = (p ^ hash_const_B) & MASK32 + hash_const_B = (hash_const_B * MULT_B) & MASK32 + state = (state * hash_const_B) & MASK32 + state = (state ^ (state >> XSHIFT)) & MASK32 + states.append(state) + + return states + + def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, auto_collate_batch, collate_fn, init_fn, worker_id, num_workers, use_shared_memory): @@ -181,6 +264,15 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, # set signal handler core._set_process_signal_handler() + # set different numpy seed for each worker + try: + import numpy as np + import time + except ImportError: + pass + else: + np.random.seed(_generate_states(int(time.time()), worker_id)) + global _worker_info _worker_info = WorkerInfo( id=worker_id, num_workers=num_workers, dataset=dataset) diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py index 977882543a888..4c69d003d80f8 100755 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py @@ -330,5 +330,19 @@ def test_main(self): self.run_main(num_workers) +class TestDataLoaderGenerateStates(unittest.TestCase): + def setUp(self): + self.inputs = [(0, 1), (0, 2), (1, 3)] + self.outputs = [[1835504127, 1731038949, 1320224556, 2330041505], + [2834126987, 2358157858, 1860244682, 1437227251], + [457190280, 2660306227, 859341110, 354512857]] + + def test_main(self): + from paddle.fluid.dataloader.worker import _generate_states + for inp, outp in zip(self.inputs, self.outputs): + out = _generate_states(*inp) + assert out == outp + + if __name__ == '__main__': unittest.main() From dfa05dac1419b1bb0e73a86da725f2669a423163 Mon Sep 17 00:00:00 2001 From: Zhang Ting Date: Thu, 10 Jun 2021 19:02:45 +0800 Subject: [PATCH 095/156] [cherry-pick] fuse L2Decay and momentum when param.regularizer is set (#32845) (#32881) fuse L2Decay and momentum when param.regularizer is set cherry-pick #32845 --- python/paddle/fluid/optimizer.py | 100 ++++++++++++++++-- python/paddle/fluid/regularizer.py | 86 --------------- .../fluid/tests/unittests/test_momentum_op.py | 71 +++++++++++++ .../fluid/tests/unittests/test_regularizer.py | 2 + python/paddle/optimizer/momentum.py | 35 +++++- python/paddle/optimizer/optimizer.py | 96 ++++++++++++++++- 6 files changed, 288 insertions(+), 102 deletions(-) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index cf2048b38b53f..9f000b2a37e31 100755 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -33,7 +33,6 @@ from .initializer import Constant from .layer_helper import LayerHelper from .layers import ops -from .regularizer import append_regularization_ops from .dygraph import base as imperative_base from .dygraph import no_grad from .dygraph.learning_rate_scheduler import LearningRateDecay, _LearningRateEpochDecay @@ -805,6 +804,93 @@ def backward(self, act_no_grad_set, callbacks) return params_grads + def _create_regularization_of_grad(self, param, grad, regularization=None): + """ Create and add backward regularization Operators + + Function helper of append_regularization_ops. + """ + # If no gradient or no regularization is specified, then we don't need to do anything + if grad is None or ((not hasattr(param, 'regularizer') or + (hasattr(param, 'regularizer') and + param.regularizer is None)) and + regularization is None): + return grad + regularization_term = None + if hasattr(param, 'regularizer') and param.regularizer is not None: + # Add variable for regularization term in grad block + regularization_term = param.regularizer(param, grad, grad.block) + elif regularization is not None: + regularization_term = regularization(param, grad, grad.block) + + assert regularization_term is not None + + new_grad = grad + if grad.type == core.VarDesc.VarType.SELECTED_ROWS: + # FIXME(zcd): If the grad is SELECTED_ROWS, after regularization, + # the grad's type and name will be changed. But the gradient's name + # is used in ParallelExecutor Reduce mode, so I add a flag for + # the new_grad here. + new_grad = grad.block.create_var( + name=grad.name + core.kNewGradSuffix(), + dtype=param.dtype, + shape=param.shape, + lod_level=param.lod_level, + type=core.VarDesc.VarType.LOD_TENSOR) + + inputs = {"X": [grad, regularization_term]} + outputs = {"Out": [new_grad]} + if framework.in_dygraph_mode(): + new_grad = core.ops.sum([grad, regularization_term]) + else: + grad.block.append_op(type='sum', inputs=inputs, outputs=outputs) + + return new_grad + + def append_regularization_ops(self, + parameters_and_grads, + regularization=None): + r"""Create and add backward regularization Operators + + Creates and adds backward regularization operators in the BlockDesc. + This will add gradients of the regularizer function to the gradients + of the parameters and return these modified gradients. This is the + same as implementing weight decay in optimizers for regularization. + + Args: + parameters_and_grads: A list of (parameters, gradients) pairs + that need to be regularized. + regularization: A global regularizer. If the parameter is not + set. It will be applied with regularizer. + + Returns: + list[(Variable, Variable)]: list of (parameters, gradients) \ + pair with the regularized gradient + + Raises: + Exception: Unknown regularization type + """ + params_and_grads = [] + if framework.in_dygraph_mode(): + for param, grad in parameters_and_grads: + new_grad = self._create_regularization_of_grad(param, grad, + regularization) + params_and_grads.append((param, new_grad)) + else: + repeate_regularizer = False + with framework.name_scope('regularization'): + for param, grad in parameters_and_grads: + if not repeate_regularizer and param.regularizer is not None and regularization is not None: + repeate_regularizer = True + logging.info( + "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " + "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" + % regularization.__str__()) + with param.block.program._optimized_guard([param, grad]): + new_grad = self._create_regularization_of_grad( + param, grad, regularization) + params_and_grads.append((param, new_grad)) + return params_and_grads + def apply_gradients(self, params_grads): """ Second part of `minimize`, appending optimization operators for @@ -837,8 +923,8 @@ def apply_gradients(self, params_grads): params_grads = append_gradient_clip_ops(params_grads) # Add regularization if any - params_grads = append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops(params_grads, + self.regularization) optimize_ops = self._create_optimization_pass(params_grads) return optimize_ops @@ -860,8 +946,8 @@ def apply_optimize(self, loss, startup_program, params_grads): framework.default_startup_program()): if self._grad_clip is not None: params_grads = self._grad_clip(params_grads) - params_grads = append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops( + params_grads, self.regularization) optimize_ops = self._create_optimization_pass(params_grads) else: program = loss.block.program @@ -1595,8 +1681,8 @@ def apply_gradients(self, params_grads): not_dgc_params_grads = append_gradient_clip_ops( not_dgc_params_grads) - not_dgc_params_grads = append_regularization_ops(not_dgc_params_grads, - self.regularization) + not_dgc_params_grads = self.append_regularization_ops( + not_dgc_params_grads, self.regularization) params_grads = not_dgc_params_grads + dgc_params_grads params_grads = sorted(params_grads, key=lambda x: x[0].name) diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index 64ce283a63c5b..64bbca6c57c54 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -22,92 +22,6 @@ __all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer'] -def _create_regularization_of_grad(param, grad, regularization=None): - """ Create and add backward regularization Operators - - Function helper of append_regularization_ops. - """ - # If no gradient or no regularization is specified, then we don't need to do anything - if grad is None or ((not hasattr(param, 'regularizer') or ( - hasattr(param, 'regularizer') and param.regularizer is None)) and - regularization is None): - return grad - regularization_term = None - if hasattr(param, 'regularizer') and param.regularizer is not None: - # Add variable for regularization term in grad block - regularization_term = param.regularizer(param, grad, grad.block) - elif regularization is not None: - regularization_term = regularization(param, grad, grad.block) - - assert regularization_term is not None - - new_grad = grad - if grad.type == core.VarDesc.VarType.SELECTED_ROWS: - # FIXME(zcd): If the grad is SELECTED_ROWS, after regularization, - # the grad's type and name will be changed. But the gradient's name - # is used in ParallelExecutor Reduce mode, so I add a flag for - # the new_grad here. - new_grad = grad.block.create_var( - name=grad.name + core.kNewGradSuffix(), - dtype=param.dtype, - shape=param.shape, - lod_level=param.lod_level, - type=core.VarDesc.VarType.LOD_TENSOR) - - inputs = {"X": [grad, regularization_term]} - outputs = {"Out": [new_grad]} - if in_dygraph_mode(): - new_grad = core.ops.sum([grad, regularization_term]) - else: - grad.block.append_op(type='sum', inputs=inputs, outputs=outputs) - - return new_grad - - -def append_regularization_ops(parameters_and_grads, regularization=None): - r"""Create and add backward regularization Operators - - Creates and adds backward regularization operators in the BlockDesc. - This will add gradients of the regularizer function to the gradients - of the parameters and return these modified gradients. This is the - same as implementing weight decay in optimizers for regularization. - - Args: - parameters_and_grads: A list of (parameters, gradients) pairs - that need to be regularized. - regularization: A global regularizer. If the parameter is not - set. It will be applied with regularizer. - - Returns: - list[(Variable, Variable)]: list of (parameters, gradients) \ - pair with the regularized gradient - - Raises: - Exception: Unknown regularization type - """ - params_and_grads = [] - if in_dygraph_mode(): - for param, grad in parameters_and_grads: - new_grad = _create_regularization_of_grad(param, grad, - regularization) - params_and_grads.append((param, new_grad)) - else: - repeate_regularizer = False - with framework.name_scope('regularization'): - for param, grad in parameters_and_grads: - if not repeate_regularizer and param.regularizer is not None and regularization is not None: - repeate_regularizer = True - logging.info( - "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " - "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % regularization.__str__()) - with param.block.program._optimized_guard([param, grad]): - new_grad = _create_regularization_of_grad(param, grad, - regularization) - params_and_grads.append((param, new_grad)) - return params_and_grads - - class WeightDecayRegularizer(object): """Base class for weight decay regularizers diff --git a/python/paddle/fluid/tests/unittests/test_momentum_op.py b/python/paddle/fluid/tests/unittests/test_momentum_op.py index 8f629b1522428..0a29e14da8c00 100644 --- a/python/paddle/fluid/tests/unittests/test_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_momentum_op.py @@ -555,6 +555,77 @@ def test_momentum_static(self): exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list) +class TestFusedMomentumWithDecayAPI(unittest.TestCase): + def get_program(self, weight_attr, bias_attr=False): + main_program = paddle.static.Program() + startup_program = paddle.static.Program() + with paddle.static.program_guard( + main_program=main_program, startup_program=startup_program): + x = paddle.static.data(name='x', shape=[10, 10]) + linear = paddle.nn.Linear( + 10, 10, weight_attr=weight_attr, bias_attr=bias_attr) + out = linear(x) + loss = paddle.mean(out) + optimizer = paddle.optimizer.Momentum( + learning_rate=0.01, + momentum=0.9, + weight_decay=paddle.regularizer.L2Decay(0.5)) + optimizer.minimize(loss) + return main_program + + def test_param_has_l2decay(self): + paddle.enable_static() + weight_attr = paddle.ParamAttr( + name="weight", + initializer=paddle.nn.initializer.Constant(value=0.5), + regularizer=paddle.regularizer.L2Decay(0.1)) + program = self.get_program(weight_attr, bias_attr=False) + ops = program.global_block().ops + + self.assertEqual(ops[-1].attr('regularization_method'), 'l2_decay') + self.assertEqual(ops[-1].attr('regularization_coeff'), np.float32(0.1)) + for i in range(len(ops)): + self.assertTrue('sum' not in ops[i].type) + self.assertTrue('scale' not in ops[i].type) + + def test_param_has_l1decay(self): + paddle.enable_static() + weight_attr = paddle.ParamAttr( + name="weight", + initializer=paddle.nn.initializer.Constant(value=0.5), + regularizer=paddle.regularizer.L1Decay(0.1)) + bias_attr = paddle.ParamAttr( + name="bias", + initializer=paddle.nn.initializer.Constant(value=0.), + regularizer=None) + program = self.get_program(weight_attr, bias_attr) + ops = program.global_block().ops + + self.assertEqual(ops[-1].type, 'momentum') + self.assertEqual(ops[-2].type, 'momentum') + self.assertEqual(ops[-3].type, 'sum') + self.assertEqual(ops[-4].type, 'scale') + self.assertEqual(ops[-5].type, 'sign') + self.assertEqual(ops[-6].type, 'matmul_grad') + if 'weight' in ops[-1].input('Param'): + self.assertEqual(ops[-1].attr('regularization_method'), '') + self.assertEqual(ops[-1].attr('regularization_coeff'), 0) + if 'bias' in ops[-2].input('Param'): + self.assertEqual(ops[-2].attr('regularization_method'), 'l2_decay') + self.assertEqual(ops[-2].attr('regularization_coeff'), + np.float32(0.5)) + + def test_param_has_no_regularizer(self): + paddle.enable_static() + program = self.get_program(weight_attr=None) + ops = program.global_block().ops + self.assertEqual(ops[-1].attr('regularization_method'), 'l2_decay') + self.assertEqual(ops[-1].attr('regularization_coeff'), np.float32(0.5)) + for i in range(len(ops)): + self.assertTrue('sum' not in ops[i].type) + self.assertTrue('scale' not in ops[i].type) + + class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): def __update_params(self, momentum, linear): for i in range(10): diff --git a/python/paddle/fluid/tests/unittests/test_regularizer.py b/python/paddle/fluid/tests/unittests/test_regularizer.py index edd69d67aaf4b..08a70fe1852d0 100644 --- a/python/paddle/fluid/tests/unittests/test_regularizer.py +++ b/python/paddle/fluid/tests/unittests/test_regularizer.py @@ -59,6 +59,7 @@ def test_l2decay_regularizer(self): params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) count_ops = len(block.ops) + optimizer = paddle.optimizer.Adam() params_grads = optimizer.append_regularization_ops(params_grads) self.assertEqual(len(params_grads), 1) self.assertEqual(len(block.ops), count_ops + 2) @@ -97,6 +98,7 @@ def test_l2decay_regularizer(self): params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) count_ops = len(block.ops) + optimizer = paddle.optimizer.Adam() params_grads = optimizer.append_regularization_ops(params_grads) self.assertEqual(len(params_grads), 1) self.assertEqual(len(block.ops), count_ops + 3) diff --git a/python/paddle/optimizer/momentum.py b/python/paddle/optimizer/momentum.py index 372143553e0c3..eb6fbb65388b2 100644 --- a/python/paddle/optimizer/momentum.py +++ b/python/paddle/optimizer/momentum.py @@ -195,6 +195,19 @@ def _create_accumulators(self, block, parameters): ) self._add_accumulator(self._velocity_acc_str, p) + def _create_regularization_of_grad(self, param, grad, regularization=None): + """ Create and add backward regularization Operators + + Function helper of append_regularization_ops. + """ + # If ParamAttr is set to L2Decay, we skip doing regularization here. And then we fused + # L2Decay with momentum which can refer to _append_optimize_op below. + if hasattr(param, 'regularizer') and isinstance(param.regularizer, + L2DecayRegularizer): + return grad + return super(Momentum, self)._create_regularization_of_grad( + param, grad, regularization) + def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -202,13 +215,27 @@ def _append_optimize_op(self, block, param_and_grad): param_and_grad[0]) lr = self._create_param_lr(param_and_grad) + # For fusion of momentum and l2decay + param = param_and_grad[0] + regularization_method = self._regularization_method + regularization_coeff = self._regularization_coeff + if hasattr(param, 'regularizer'): + # we skip param's l2decay before, so fuse it with momentum here. + if isinstance(param.regularizer, L2DecayRegularizer): + regularization_method = "l2_decay" + regularization_coeff = param.regularizer._regularization_coeff + # the param's regularization has been done before, we avoid do l2decay in momentum. + elif param.regularizer is not None: + regularization_method = "" + regularization_coeff = 0 + if framework.in_dygraph_mode(): _, _ = core.ops.momentum( param_and_grad[0], param_and_grad[1], velocity_acc, lr, param_and_grad[0], velocity_acc, 'mu', self._momentum, 'use_nesterov', self._use_nesterov, 'regularization_method', - self._regularization_method, 'regularization_coeff', - self._regularization_coeff) + regularization_method, 'regularization_coeff', + regularization_coeff) return None find_master = self._multi_precision and param_and_grad[ @@ -219,8 +246,8 @@ def _append_optimize_op(self, block, param_and_grad): attrs = { "mu": self._momentum, "use_nesterov": self._use_nesterov, - "regularization_method": self._regularization_method, - "regularization_coeff": self._regularization_coeff, + "regularization_method": regularization_method, + "regularization_coeff": regularization_coeff, "multi_precision": find_master, "rescale_grad": self._rescale_grad } diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index b06bd2a2b0be9..8615059b06df5 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -32,7 +32,6 @@ from ..fluid.initializer import Constant from ..fluid.layer_helper import LayerHelper from ..fluid.layers import ops -from ..fluid.regularizer import append_regularization_ops from ..fluid.dygraph import base as imperative_base from ..fluid.dygraph import no_grad from paddle.fluid import core @@ -769,8 +768,8 @@ def apply_gradients(self, params_grads): params_grads = append_gradient_clip_ops(params_grads) # Add regularization if any - params_grads = append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops(params_grads, + self.regularization) optimize_ops = self._create_optimization_pass(params_grads) return optimize_ops @@ -792,8 +791,8 @@ def _apply_optimize(self, loss, startup_program, params_grads): framework.default_startup_program()): if self._grad_clip is not None: params_grads = self._grad_clip(params_grads) - params_grads = append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops( + params_grads, self.regularization) optimize_ops = self._create_optimization_pass(params_grads) else: program = loss.block.program @@ -801,6 +800,93 @@ def _apply_optimize(self, loss, startup_program, params_grads): optimize_ops = self.apply_gradients(params_grads) return optimize_ops + def _create_regularization_of_grad(self, param, grad, regularization=None): + """ Create and add backward regularization Operators + + Function helper of append_regularization_ops. + """ + # If no gradient or no regularization is specified, then we don't need to do anything + if grad is None or ((not hasattr(param, 'regularizer') or + (hasattr(param, 'regularizer') and + param.regularizer is None)) and + regularization is None): + return grad + regularization_term = None + if hasattr(param, 'regularizer') and param.regularizer is not None: + # Add variable for regularization term in grad block + regularization_term = param.regularizer(param, grad, grad.block) + elif regularization is not None: + regularization_term = regularization(param, grad, grad.block) + + assert regularization_term is not None + + new_grad = grad + if grad.type == core.VarDesc.VarType.SELECTED_ROWS: + # FIXME(zcd): If the grad is SELECTED_ROWS, after regularization, + # the grad's type and name will be changed. But the gradient's name + # is used in ParallelExecutor Reduce mode, so I add a flag for + # the new_grad here. + new_grad = grad.block.create_var( + name=grad.name + core.kNewGradSuffix(), + dtype=param.dtype, + shape=param.shape, + lod_level=param.lod_level, + type=core.VarDesc.VarType.LOD_TENSOR) + + inputs = {"X": [grad, regularization_term]} + outputs = {"Out": [new_grad]} + if framework.in_dygraph_mode(): + new_grad = core.ops.sum([grad, regularization_term]) + else: + grad.block.append_op(type='sum', inputs=inputs, outputs=outputs) + + return new_grad + + def append_regularization_ops(self, + parameters_and_grads, + regularization=None): + r"""Create and add backward regularization Operators + + Creates and adds backward regularization operators in the BlockDesc. + This will add gradients of the regularizer function to the gradients + of the parameters and return these modified gradients. This is the + same as implementing weight decay in optimizers for regularization. + + Args: + parameters_and_grads: A list of (parameters, gradients) pairs + that need to be regularized. + regularization: A global regularizer. If the parameter is not + set. It will be applied with regularizer. + + Returns: + list[(Variable, Variable)]: list of (parameters, gradients) \ + pair with the regularized gradient + + Raises: + Exception: Unknown regularization type + """ + params_and_grads = [] + if framework.in_dygraph_mode(): + for param, grad in parameters_and_grads: + new_grad = self._create_regularization_of_grad(param, grad, + regularization) + params_and_grads.append((param, new_grad)) + else: + repeate_regularizer = False + with framework.name_scope('regularization'): + for param, grad in parameters_and_grads: + if not repeate_regularizer and param.regularizer is not None and regularization is not None: + repeate_regularizer = True + logging.info( + "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " + "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" + % regularization.__str__()) + with param.block.program._optimized_guard([param, grad]): + new_grad = self._create_regularization_of_grad( + param, grad, regularization) + params_and_grads.append((param, new_grad)) + return params_and_grads + def _get_no_grad_set(self, loss, no_grad_set=None): no_grad_set = _get_no_grad_set_name(no_grad_set) parameters = loss.block.program.global_block().all_parameters() From 8461ab17f087b41b74d66f3bbbe7ac2d24e29d59 Mon Sep 17 00:00:00 2001 From: LielinJiang <50691816+LielinJiang@users.noreply.github.com> Date: Thu, 10 Jun 2021 19:12:34 +0800 Subject: [PATCH 096/156] add sample code for summary (#33337) (#33427) --- python/paddle/hapi/model_summary.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/paddle/hapi/model_summary.py b/python/paddle/hapi/model_summary.py index d78196d94451e..93f1a5a37a67f 100644 --- a/python/paddle/hapi/model_summary.py +++ b/python/paddle/hapi/model_summary.py @@ -80,6 +80,23 @@ def forward(self, inputs): params_info = paddle.summary(lenet, (1, 1, 28, 28)) print(params_info) + # multi input demo + class LeNetMultiInput(LeNet): + + def forward(self, inputs, y): + x = self.features(inputs) + + if self.num_classes > 0: + x = paddle.flatten(x, 1) + x = self.fc(x + y) + return x + + lenet_multi_input = LeNetMultiInput() + + params_info = paddle.summary(lenet_multi_input, [(1, 1, 28, 28), (1, 400)], + ['float32', 'float32']) + print(params_info) + """ if isinstance(input_size, InputSpec): _input_size = tuple(input_size.shape) From 61cae0dff33a20d0af97cf2cf380ef0982181758 Mon Sep 17 00:00:00 2001 From: Lijunhui <1578034415@qq.com> Date: Fri, 11 Jun 2021 11:20:16 +0800 Subject: [PATCH 097/156] [cherry-pick]Fixed a bug of log_softmax: op input was modified to 'nan' (#32937) (#33436) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使用op benchmark时发现，当输入数据量小于某个值时，python 端 log_softmax 接口的输入值经过计算过后 会被改变为nan。输出正常。 cherry-pick自 #32937 --- paddle/fluid/operators/log_softmax_op.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/operators/log_softmax_op.cu b/paddle/fluid/operators/log_softmax_op.cu index e4fe92c625640..12c607adb44f4 100644 --- a/paddle/fluid/operators/log_softmax_op.cu +++ b/paddle/fluid/operators/log_softmax_op.cu @@ -104,7 +104,7 @@ __global__ void ComputeLogSoftmaxForwardInWarp(T *dst, const T *src, #pragma unroll for (int it = 0; it < warp_iter; ++it) { int element_index = thread_in_warp_idx + it * kernel_warp_size; - if (element_index < element_count) { + if (element_index < effective_element_count) { dst[batch_id * element_count + element_index] = static_cast(elements[it] - max_value - sum); } else { @@ -226,7 +226,7 @@ __global__ void ComputeLogSoftmaxBackwardInWarp(const T *output, #pragma unroll for (int iter = 0; iter < warp_iter; ++iter) { int element_index = thread_in_warp_idx + iter * kernel_warp_size; - if (element_index < element_count) { + if (element_index < effective_element_count) { grad_input[batch_id * element_count + element_index] = static_cast( (grad_output_register[iter] - std::exp(output_register[iter]) * sum)); } From f57ae4d7170b48d19c78251f33db6caee310cc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=20Wei=20=28=E4=BB=BB=E5=8D=AB=29?= Date: Fri, 11 Jun 2021 11:24:28 +0800 Subject: [PATCH 098/156] [cherry-pick] use the required instruction to determine if the environment fits the sample code's required. (#32766) (#33451) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 put a instruction # required: gpu(for example) in the sample code 2 this piece of code will only run in the GPU-equipped CI pipelines, and be omitted in other pipelines. 3 the CI pipelines can specify its capacity by shell environment variable SAMPLE_CODE_TEST_CAPACITY 2.1 文档改版方案 see #32766 for more infomation --- tools/check_file_diff_approvals.sh | 15 +- tools/sampcd_processor.py | 484 ++++++++++++++++++--------- tools/test_sampcd_processor.py | 402 ++++++++++++++++++----- tools/wlist.json | 505 ----------------------------- 4 files changed, 661 insertions(+), 745 deletions(-) delete mode 100644 tools/wlist.json diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh index b1395c28878e3..ef9af288fb0a2 100644 --- a/tools/check_file_diff_approvals.sh +++ b/tools/check_file_diff_approvals.sh @@ -52,7 +52,7 @@ API_FILES=("CMakeLists.txt" "python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py" "python/paddle/fluid/tests/unittests/white_list/check_op_sequence_batch_1_input_white_list.py" "python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py" - "tools/wlist.json" + "tools/print_signatures.py" "tools/sampcd_processor.py" "paddle/scripts/paddle_build.bat" "tools/windows/run_unittests.sh" @@ -80,11 +80,10 @@ function add_failed(){ echo_list="${echo_list[@]}$1" } -function run_test_sampcd_processor() { +function run_tools_test() { CUR_PWD=$(pwd) cd ${PADDLE_ROOT}/tools - python test_sampcd_processor.py - python test_print_signatures.py + python $1 cd ${CUR_PWD} } @@ -141,12 +140,12 @@ for API_FILE in ${API_FILES[*]}; do elif [ "${API_FILE}" == "python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py" ];then echo_line="You must have one RD (Shixiaowei02 (Recommend), luotao1 or phlrain) approval for the python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py, which manages the white list of no_grad_set without value in operators. For more information, please refer to[https://github.com/PaddlePaddle/Paddle/wiki/It's-recommend-to-set-no_grad_set-to-be-None].\n" check_approval 1 39303645 6836917 43953930 - elif [ "${API_FILE}" == "tools/wlist.json" ];then - echo_line="You must have one TPM (jzhang533) approval for the api whitelist for the tools/wlist.json.\n" - check_approval 1 29231 elif [ "${API_FILE}" == "tools/sampcd_processor.py" ];then echo_line="test_sampcd_processor.py will be executed for changed sampcd_processor.py.\n" - run_test_sampcd_processor + run_tools_test test_sampcd_processor.py + elif [ "${API_FILE}" == "tools/print_signatures.py" ];then + echo_line="test_print_signatures.py will be executed for changed print_signatures.py.\n" + run_tools_test test_print_signatures.py elif [ "${API_FILE}" == "python/paddle/distributed/fleet/__init__.py" ]; then echo_line="You must have (fuyinno4 (Recommend), raindrops2sea) approval for ${API_FILE} changes" check_approval 1 35824027 38231817 diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index 52777cd59ba25..a1658e3c2edf7 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -11,12 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +please make sure to run in the tools path +usage: python sample_test.py {cpu or gpu} + {cpu or gpu}: running in cpu version or gpu version + +for example, you can run cpu version python2 testing like this: + + python sampcd_processor.py cpu +""" import os import sys import subprocess import multiprocessing -import math import platform import inspect import json @@ -24,16 +32,7 @@ import shutil import re import logging -""" -please make sure to run in the tools path -usage: python sample_test.py {cpu or gpu} - {cpu or gpu}: running in cpu version or gpu version - -for example, you can run cpu version python2 testing like this: - - python sampcd_processor.py cpu - -""" +import time logger = logging.getLogger() if logger.handlers: @@ -45,6 +44,7 @@ console.setFormatter(logging.Formatter("%(message)s")) RUN_ON_DEVICE = 'cpu' +SAMPLE_CODE_TEST_CAPACITY = set() GPU_ID = 0 methods = [] whl_error = [] @@ -52,6 +52,15 @@ API_PR_SPEC_FN = 'paddle/fluid/API_PR.spec' API_DIFF_SPEC_FN = 'dev_pr_diff_api.spec' SAMPLECODE_TEMPDIR = 'samplecode_temp' +ENV_KEY_CODES_FRONTEND = 'CODES_INSERTED_INTO_FRONTEND' +ENV_KEY_TEST_CAPACITY = 'SAMPLE_CODE_TEST_CAPACITY' +SUMMARY_INFO = { + 'success': [], + 'failed': [], + 'skiptest': [], + 'nocodes': [], + # ... required not-match +} def find_all(srcstr, substr): @@ -75,32 +84,225 @@ def find_all(srcstr, substr): return indices -def check_indent(cdline): +def find_last_future_line_end(cbstr): + """ + find the last `__future__` line. + + Args: + docstr(str): docstring + Return: + index of the line end or None. """ - to check the indent of a given code line + pat = re.compile('__future__.*\n') + lastmo = None + it = re.finditer(pat, cbstr) + while True: + try: + lastmo = next(it) + except StopIteration: + break + if lastmo: + return lastmo.end() + else: + return None - to get the number of starting blank chars, - e.t. blankspaces and \t - \t will be interpreted as 4 single blankspaces, - e.t. '\t'=' ' +def extract_code_blocks_from_docstr(docstr): + """ + extract code-blocks from the given docstring. + + DON'T include the multiline-string definition in code-blocks. + The *Examples* section must be the last. Args: - cdline(str) : a single line of code from the source file + docstr(str): docstring + Return: + code_blocks: A list of code-blocks, indent removed. + element {'name': the code-block's name, 'id': sequence id. + 'codes': codes, 'required': 'gpu'} + """ + code_blocks = [] + + mo = re.search(r"Examples:", docstr) + if mo is None: + return code_blocks + ds_list = docstr[mo.start():].replace("\t", ' ').split("\n") + lastlineindex = len(ds_list) - 1 + + cb_start_pat = re.compile(r"code-block::\s*python") + cb_param_pat = re.compile(r"^\s*:(\w+):\s*(\S*)\s*$") + cb_required_pat = re.compile(r"^\s*#\s*require[s|d]\s*:\s*(\S+)\s*$") + + cb_info = {} + cb_info['cb_started'] = False + cb_info['cb_cur'] = [] + cb_info['cb_cur_indent'] = -1 + cb_info['cb_cur_name'] = None + cb_info['cb_cur_seq_id'] = 0 + cb_info['cb_required'] = None + + def _cb_started(): + # nonlocal cb_started, cb_cur_name, cb_required, cb_cur_seq_id + cb_info['cb_started'] = True + cb_info['cb_cur_seq_id'] += 1 + cb_info['cb_cur_name'] = None + cb_info['cb_required'] = None + + def _append_code_block(): + # nonlocal code_blocks, cb_cur, cb_cur_name, cb_cur_seq_id, cb_required + code_blocks.append({ + 'codes': inspect.cleandoc("\n".join(cb_info['cb_cur'])), + 'name': cb_info['cb_cur_name'], + 'id': cb_info['cb_cur_seq_id'], + 'required': cb_info['cb_required'], + }) + + for lineno, linecont in enumerate(ds_list): + if re.search(cb_start_pat, linecont): + if not cb_info['cb_started']: + _cb_started() + continue + else: + # cur block end + if len(cb_info['cb_cur']): + _append_code_block() + _cb_started() # another block started + cb_info['cb_cur_indent'] = -1 + cb_info['cb_cur'] = [] + else: + if cb_info['cb_started']: + # handle the code-block directive's options + mo_p = cb_param_pat.match(linecont) + if mo_p: + if mo_p.group(1) == 'name': + cb_info['cb_cur_name'] = mo_p.group(2) + continue + # read the required directive + mo_r = cb_required_pat.match(linecont) + if mo_r: + cb_info['cb_required'] = mo_r.group(1) + # docstring end + if lineno == lastlineindex: + mo = re.search(r"\S", linecont) + if mo is not None and cb_info['cb_cur_indent'] <= mo.start( + ): + cb_info['cb_cur'].append(linecont) + if len(cb_info['cb_cur']): + _append_code_block() + break + # check indent for cur block start and end. + mo = re.search(r"\S", linecont) + if mo is None: + continue + if cb_info['cb_cur_indent'] < 0: + # find the first non empty line + cb_info['cb_cur_indent'] = mo.start() + cb_info['cb_cur'].append(linecont) + else: + if cb_info['cb_cur_indent'] <= mo.start(): + cb_info['cb_cur'].append(linecont) + else: + if linecont[mo.start()] == '#': + continue + else: + # block end + if len(cb_info['cb_cur']): + _append_code_block() + cb_info['cb_started'] = False + cb_info['cb_cur_indent'] = -1 + cb_info['cb_cur'] = [] + return code_blocks + + +def get_test_capacity(): + """ + collect capacities and set to SAMPLE_CODE_TEST_CAPACITY + """ + global SAMPLE_CODE_TEST_CAPACITY # write + global ENV_KEY_TEST_CAPACITY, RUN_ON_DEVICE # readonly + if ENV_KEY_TEST_CAPACITY in os.environ: + for r in os.environ[ENV_KEY_TEST_CAPACITY].split(','): + rr = r.strip().lower() + if r: + SAMPLE_CODE_TEST_CAPACITY.add(rr) + if 'cpu' not in SAMPLE_CODE_TEST_CAPACITY: + SAMPLE_CODE_TEST_CAPACITY.add('cpu') - Returns: - int : the indent of the number of interpreted - blankspaces + if RUN_ON_DEVICE: + SAMPLE_CODE_TEST_CAPACITY.add(RUN_ON_DEVICE) + + +def is_required_match(requirestr, cbtitle='not-specified'): """ - indent = 0 - for c in cdline: - if c == '\t': - indent += 4 - elif c == ' ': - indent += 1 - if c != ' ' and c != '\t': - break - return indent + search the required instruction in the code-block, and check it match the current running environment. + + environment values of equipped: cpu, gpu, xpu, distributed, skip + the 'skip' is the special flag to skip the test, so is_required_match will return False directly. + + Args: + requirestr(str): the required string. + cbtitle(str): the title of the code-block. + returns: + True - yes, matched + False - not match + None - skipped # trick + """ + global SAMPLE_CODE_TEST_CAPACITY # readonly + requires = set(['cpu']) + if requirestr: + for r in requirestr.split(','): + rr = r.strip().lower() + if rr: + requires.add(rr) + if 'skip' in requires or 'skiptest' in requires: + logger.info('%s: skipped', cbtitle) + return None + + if all([ + k in SAMPLE_CODE_TEST_CAPACITY for k in requires + if k not in ['skip', 'skiptest'] + ]): + return True + + logger.info('%s: the equipments [%s] not match the required [%s].', cbtitle, + ','.join(SAMPLE_CODE_TEST_CAPACITY), ','.join(requires)) + return False + + +def insert_codes_into_codeblock(codeblock, apiname='not-specified'): + """ + insert some codes in the frontend and backend into the code-block. + """ + global ENV_KEY_CODES_FRONTEND, GPU_ID, RUN_ON_DEVICE # readonly + inserted_codes_f = '' + inserted_codes_b = '' + if ENV_KEY_CODES_FRONTEND in os.environ and os.environ[ + ENV_KEY_CODES_FRONTEND]: + inserted_codes_f = os.environ[ENV_KEY_CODES_FRONTEND] + else: + cpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = ""\n' + gpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = "{}"\n'.format( + GPU_ID) + if 'required' in codeblock: + if codeblock['required'] is None or codeblock['required'] == 'cpu': + inserted_codes_f = cpu_str + elif codeblock['required'] == 'gpu': + inserted_codes_f = gpu_str + else: + if RUN_ON_DEVICE == "cpu": + inserted_codes_f = cpu_str + elif RUN_ON_DEVICE == "gpu": + inserted_codes_f = gpu_str + inserted_codes_b = '\nprint("{}\'s sample code (name:{}, id:{}) is executed successfully!")'.format( + apiname, codeblock['name'], codeblock['id']) + + cb = codeblock['codes'] + last_future_line_end = find_last_future_line_end(cb) + if last_future_line_end: + return cb[:last_future_line_end] + inserted_codes_f + cb[ + last_future_line_end:] + inserted_codes_b + else: + return inserted_codes_f + cb + inserted_codes_b def sampcd_extract_to_file(srccom, name, htype="def", hname=""): @@ -117,122 +319,111 @@ def sampcd_extract_to_file(srccom, name, htype="def", hname=""): Returns: sample_code_filenames(list of str) """ - global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR - CODE_BLOCK_INTERDUCTORY = "code-block:: python" + global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR # readonly + global SUMMARY_INFO # update - sampcd_begins = find_all(srccom, CODE_BLOCK_INTERDUCTORY) - if len(sampcd_begins) == 0: + codeblocks = extract_code_blocks_from_docstr(srccom) + if len(codeblocks) == 0: + SUMMARY_INFO['nocodes'].append(name) # detect sample codes using >>> to format and consider this situation as wrong - print(htype, " name:", hname) - print("-----------------------") + logger.info(htype + " name:" + name) + logger.info("-----------------------") if srccom.find("Examples:") != -1: - print("----example code check----\n") + logger.info("----example code check----") if srccom.find(">>>") != -1: - print( - "Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n ", - "Please use '.. code-block:: python' to ", - "format sample code.\n") + logger.warning(r"""Deprecated sample code style: + Examples: + >>>codeline + >>>codeline + +Please use '.. code-block:: python' to format the sample code.""") return [] else: - print("Error: No sample code!\n") + logger.warning("Error: No sample code!") return [] + sample_code_filenames = [] - for y in range(1, len(sampcd_begins) + 1): - sampcd_begin = sampcd_begins[y - 1] - sampcd = srccom[sampcd_begin + len(CODE_BLOCK_INTERDUCTORY) + 1:] - sampcd = sampcd.split("\n") - # remove starting empty lines - while sampcd[0].replace(' ', '').replace('\t', '') == '': - sampcd.pop(0) - - # the minimum indent, which is the indent of the first - # non-empty line - min_indent = check_indent(sampcd[0]) - sampcd_to_write = [] - for i in range(0, len(sampcd)): - cdline = sampcd[i] - # handle empty lines or those only with spaces/tabs - if cdline.strip() == '': - continue - this_indent = check_indent(cdline) - if this_indent < min_indent: - break - else: - cdline = cdline.replace('\t', ' ') - sampcd_to_write.append(cdline[min_indent:]) - - sampcd = '\n'.join(sampcd_to_write) - if RUN_ON_DEVICE == "cpu": - sampcd = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = ""\n' + sampcd - if RUN_ON_DEVICE == "gpu": - sampcd = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = "{}"\n'.format( - GPU_ID) + sampcd - sampcd += '\nprint(' + '\"' + name + ' sample code is executed successfully!\")' - - tfname = os.path.join(SAMPLECODE_TEMPDIR, '{}_example{}'.format( - name, '.py' if len(sampcd_begins) == 1 else '_{}.py'.format(y))) - with open(tfname, 'w') as tempf: - tempf.write(sampcd) - sample_code_filenames.append(tfname) + for y, cb in enumerate(codeblocks): + matched = is_required_match(cb['required'], name) + # matched has three states: + # True - please execute it; + # None - no sample code found; + # False - it need other special equipment or environment. + # so, the following conditional statements are intentionally arranged. + if matched == True: + tfname = os.path.join(SAMPLECODE_TEMPDIR, '{}_example{}'.format( + name, '.py' + if len(codeblocks) == 1 else '_{}.py'.format(y + 1))) + with open(tfname, 'w') as tempf: + sampcd = insert_codes_into_codeblock(cb, name) + tempf.write(sampcd) + sample_code_filenames.append(tfname) + elif matched is None: + logger.info('{}\' code block (name:{}, id:{}) is skipped.'.format( + name, cb['name'], cb['id'])) + SUMMARY_INFO['skiptest'].append("{}-{}".format(name, cb['id'])) + elif matched == False: + logger.info( + '{}\' code block (name:{}, id:{}) required({}) not match capacity({}).'. + format(name, cb['name'], cb['id'], cb['required'], + SAMPLE_CODE_TEST_CAPACITY)) + if cb['required'] not in SUMMARY_INFO: + SUMMARY_INFO[cb['required']] = [] + SUMMARY_INFO[cb['required']].append("{}-{}".format(name, cb['id'])) + return sample_code_filenames def execute_samplecode(tfname): """ - Execute a sample-code test. + Execute a sample-code test Args: - tfname: the filename of the samplecode. + tfname: the filename of the sample code Returns: result: success or not tfname: same as the input argument - msg: the stdout output of the samplecode executing. + msg: the stdout output of the sample code executing + time: time consumed by sample code """ result = True msg = None if platform.python_version()[0] in ["2", "3"]: cmd = [sys.executable, tfname] else: - print("Error: fail to parse python version!") + logger.error("Error: fail to parse python version!") result = False exit(1) - # check required envisonment - with open(tfname, 'r') as f: - for line in f.readlines(): - if re.match(r'#\s*required\s*:\s*(distributed|gpu|skip)', line): - result = True - return result, tfname, '{} is skipped. cause: {}'.format(tfname, - line) - - logging.info('running %s', tfname) - print("\n----example code check----") - print("executing sample code .....", tfname) + logger.info("----example code check----") + logger.info("executing sample code: %s", tfname) + start_time = time.time() subprc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = subprc.communicate() msg = "".join(output.decode(encoding='utf-8')) err = "".join(error.decode(encoding='utf-8')) + end_time = time.time() if subprc.returncode != 0: - print("Sample code error found in ", tfname, ":") - print("-----------------------") - print(open(tfname).read()) - print("-----------------------") - print("subprocess return code: ", str(subprc.returncode)) - print("Error Raised from Sample Code ", tfname, " :") - print(err) - print(msg) - print("----example code check failed----\n") - logging.warning('%s error: %s', tfname, err) - logging.warning('%s msg: %s', tfname, msg) + with open(tfname, 'r') as f: + logger.warning("""Sample code error found in %s: +----------------------- +%s +----------------------- +subprocess return code: %d +Error Raised from Sample Code: +stderr: %s +stdout: %s +""", tfname, f.read(), subprc.returncode, err, msg) + logger.info("----example code check failed----") result = False else: - print("----example code check success----\n") + logger.info("----example code check success----") # msg is the returned code execution report - return result, tfname, msg + return result, tfname, msg, end_time - start_time def get_filenames(): @@ -317,35 +508,6 @@ def get_incrementapi(): f.write('\n') -def get_wlist(fn="wlist.json"): - ''' - this function will get the white list of API. - - Returns: - - wlist: a list of API that should not trigger the example check . - - ''' - wlist = [] - wlist_file = [] - # only white on CPU - gpu_not_white = [] - with open(fn, 'r') as load_f: - load_dict = json.load(load_f) - for key in load_dict: - if key == 'wlist_dir': - for item in load_dict[key]: - wlist_file.append(item["name"]) - elif key == "gpu_not_white": - gpu_not_white = load_dict[key] - elif key == "wlist_api": - for item in load_dict[key]: - wlist.append(item["name"]) - else: - wlist = wlist + load_dict[key] - return wlist, wlist_file, gpu_not_white - - arguments = [ # flags, dest, type, default, help ['--gpu_id', 'gpu_id', int, 0, 'GPU device id to use [0]'], @@ -391,18 +553,15 @@ def parse_args(): )) logger.addHandler(logfHandler) - wlist, wlist_file, gpu_not_white = get_wlist() - if args.mode == "gpu": GPU_ID = args.gpu_id logger.info("using GPU_ID %d", GPU_ID) - for _gnw in gpu_not_white: - wlist.remove(_gnw) elif args.mode != "cpu": logger.error("Unrecognized argument:%s, 'cpu' or 'gpu' is desired.", args.mode) sys.exit("Invalid arguments") RUN_ON_DEVICE = args.mode + get_test_capacity() logger.info("API check -- Example Code") logger.info("sample_test running under python %s", platform.python_version()) @@ -449,19 +608,50 @@ def parse_args(): if not temp[0]: logger.info("In addition, mistakes found in sample codes: %s", temp[1]) - logger.info("error_methods: %s", str(temp[2])) logger.info("----------------------------------------------------") exit(1) else: - has_error = False + timeovered_test = {} for temp in result: if not temp[0]: logger.info("In addition, mistakes found in sample codes: %s", temp[1]) - logger.info("error_methods: %s", str(temp[2])) - has_error = True - if has_error: - logger.info("Mistakes found in sample codes.") - logger.info("Please check sample codes.") + SUMMARY_INFO['failed'].append(temp[1]) + else: + SUMMARY_INFO['success'].append(temp[1]) + if temp[3] > 10: + timeovered_test[temp[1]] = temp[3] + + if len(timeovered_test): + logger.info("%d sample codes ran time over 10s", + len(timeovered_test)) + if args.debug: + for k, v in timeovered_test.items(): + logger.info('{} - {}s'.format(k, v)) + if len(SUMMARY_INFO['success']): + logger.info("%d sample codes ran success", + len(SUMMARY_INFO['success'])) + for k, v in SUMMARY_INFO.items(): + if k not in ['success', 'failed', 'skiptest', 'nocodes']: + logger.info("%d sample codes required not match for %s", + len(v), k) + if len(SUMMARY_INFO['skiptest']): + logger.info("%d sample codes skipped", + len(SUMMARY_INFO['skiptest'])) + if args.debug: + logger.info('\n'.join(SUMMARY_INFO['skiptest'])) + if len(SUMMARY_INFO['nocodes']): + logger.info("%d apis don't have sample codes", + len(SUMMARY_INFO['nocodes'])) + if args.debug: + logger.info('\n'.join(SUMMARY_INFO['nocodes'])) + if len(SUMMARY_INFO['failed']): + logger.info("%d sample codes ran failed", + len(SUMMARY_INFO['failed'])) + logger.info('\n'.join(SUMMARY_INFO['failed'])) + logger.info( + "Mistakes found in sample codes. Please recheck the sample codes." + ) exit(1) + logger.info("Sample code check is successful!") diff --git a/tools/test_sampcd_processor.py b/tools/test_sampcd_processor.py index 7836728247f50..81710dae16764 100644 --- a/tools/test_sampcd_processor.py +++ b/tools/test_sampcd_processor.py @@ -20,15 +20,18 @@ import shutil import sys import importlib +import re +import sampcd_processor from sampcd_processor import find_all -from sampcd_processor import check_indent from sampcd_processor import get_api_md5 from sampcd_processor import get_incrementapi -from sampcd_processor import get_wlist from sampcd_processor import sampcd_extract_to_file +from sampcd_processor import extract_code_blocks_from_docstr from sampcd_processor import execute_samplecode - -SAMPLECODE_TEMP_DIR = 'samplecode_temp' +from sampcd_processor import find_last_future_line_end +from sampcd_processor import insert_codes_into_codeblock +from sampcd_processor import get_test_capacity +from sampcd_processor import is_required_match class Test_find_all(unittest.TestCase): @@ -43,27 +46,246 @@ def test_find_two(self): find_all(' hello, world; hello paddle!', 'hello')) -class Test_check_indent(unittest.TestCase): - def test_no_indent(self): - self.assertEqual(0, check_indent('hello paddle')) +class Test_find_last_future_line_end(unittest.TestCase): + def test_no_instant(self): + samplecodes = """ + print(10//3) + """ + self.assertIsNone(find_last_future_line_end(samplecodes)) + + def test_1_instant(self): + samplecodes = """ + from __future__ import print_function + + print(10//3) + """ + mo = re.search("print_function\n", samplecodes) + self.assertIsNotNone(mo) + self.assertGreaterEqual( + find_last_future_line_end(samplecodes), mo.end()) + + def test_2_instant(self): + samplecodes = """ + from __future__ import print_function + from __future__ import division + + print(10//3) + """ + mo = re.search("division\n", samplecodes) + self.assertIsNotNone(mo) + self.assertGreaterEqual( + find_last_future_line_end(samplecodes), mo.end()) + + +class Test_extract_code_blocks_from_docstr(unittest.TestCase): + def test_no_samplecode(self): + docstr = """ + placeholder + """ + codeblocks = extract_code_blocks_from_docstr(docstr) + self.assertListEqual([], codeblocks) + + def test_codeblock_before_examples_is_ignored(self): + docstr = """ + .. code-block:: python + + print(1+1) + Examples: + """ + codeblocks = extract_code_blocks_from_docstr(docstr) + self.assertListEqual(codeblocks, []) + + def test_1_samplecode(self): + docstr = """ + Examples: + .. code-block:: python + + print(1+1) + """ + codeblocks = extract_code_blocks_from_docstr(docstr) + self.assertListEqual(codeblocks, [{ + 'codes': """print(1+1)""", + 'name': None, + 'id': 1, + 'required': None, + }]) + + def test_2_samplecodes(self): + docstr = """ + placeholder + Examples: + .. code-block:: python + + print(1/0) + + .. code-block:: python + :name: one_plus_one + :linenos: + + # required: gpu + print(1+1) + """ + codeblocks = extract_code_blocks_from_docstr(docstr) + self.assertListEqual(codeblocks, [{ + 'codes': """print(1/0)""", + 'name': None, + 'id': 1, + 'required': None, + }, { + 'codes': """# required: gpu +print(1+1)""", + 'name': 'one_plus_one', + 'id': 2, + 'required': 'gpu', + }]) + + +class Test_insert_codes_into_codeblock(unittest.TestCase): + def test_required_None(self): + codeblock = { + 'codes': """print(1/0)""", + 'name': None, + 'id': 1, + 'required': None, + } + self.assertEqual(""" +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "" +print(1/0) +print("not-specified's sample code (name:None, id:1) is executed successfully!")""", + insert_codes_into_codeblock(codeblock)) + + def test_required_gpu(self): + codeblock = { + 'codes': """# required: gpu +print(1+1)""", + 'name': None, + 'id': 1, + 'required': 'gpu', + } + self.assertEqual(""" +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "0" +# required: gpu +print(1+1) +print("not-specified's sample code (name:None, id:1) is executed successfully!")""", + insert_codes_into_codeblock(codeblock)) + + def test_from_future(self): + codeblock = { + 'codes': """ +from __future__ import print_function +from __future__ import division +print(10//3)""", + 'name': 'future', + 'id': 1, + 'required': None, + } + self.assertEqual(""" +from __future__ import print_function +from __future__ import division + +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "" +print(10//3) +print("not-specified's sample code (name:future, id:1) is executed successfully!")""", + insert_codes_into_codeblock(codeblock)) + + +def clear_capacity(): + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY = set() + sampcd_processor.RUN_ON_DEVICE = 'cpu' + if sampcd_processor.ENV_KEY_TEST_CAPACITY in os.environ: + del os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] - def test_indent_4_spaces(self): - self.assertEqual(4, check_indent(' hello paddle')) - def test_indent_1_tab(self): - self.assertEqual(4, check_indent("\thello paddle")) +class Test_get_test_capacity(unittest.TestCase): + def setUp(self): + clear_capacity() + get_test_capacity() + + def tearDown(self): + clear_capacity() + get_test_capacity() + + def test_NoEnvVar(self): + clear_capacity() + get_test_capacity() + self.assertCountEqual(['cpu', ], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + + def test_NoEnvVar_RUN_ON_DEVICE_gpu(self): + clear_capacity() + sampcd_processor.RUN_ON_DEVICE = 'gpu' + get_test_capacity() + self.assertCountEqual(['cpu', 'gpu'], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + + def test_EnvVar_gpu(self): + clear_capacity() + os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu' + get_test_capacity() + self.assertCountEqual(['cpu', 'gpu'], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + + def test_EnvVar_gpu_and_distributed(self): + clear_capacity() + os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu,distributed' + get_test_capacity() + self.assertCountEqual(['cpu', 'gpu', 'distributed'], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + + +class Test_is_required_match(unittest.TestCase): + def setUp(self): + clear_capacity() + + def tearDown(self): + clear_capacity() + get_test_capacity() + + def test_alldefault(self): + clear_capacity() + get_test_capacity() + self.assertTrue(is_required_match('')) + self.assertTrue(is_required_match(None)) + self.assertTrue(is_required_match('cpu')) + self.assertFalse(is_required_match('gpu')) + self.assertIsNone(is_required_match('skiptest')) + self.assertIsNone(is_required_match('skip')) + self.assertIsNone(is_required_match('cpu,skiptest')) + + def test_gpu_equipped(self): + clear_capacity() + os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu' + get_test_capacity() + self.assertTrue(is_required_match('cpu')) + self.assertTrue(is_required_match('gpu')) + self.assertTrue(is_required_match('gpu,cpu')) + self.assertIsNone(is_required_match('skiptest')) + self.assertFalse(is_required_match('distributed')) + + def test_gpu_distributed_equipped(self): + clear_capacity() + os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu,distributed' + get_test_capacity() + self.assertTrue(is_required_match('cpu')) + self.assertTrue(is_required_match('gpu')) + self.assertTrue(is_required_match('distributed')) + self.assertFalse(is_required_match('xpu')) + self.assertIsNone(is_required_match('skiptest')) class Test_execute_samplecode(unittest.TestCase): def setUp(self): - if not os.path.exists(SAMPLECODE_TEMP_DIR): - os.mkdir(SAMPLECODE_TEMP_DIR) - self.successSampleCodeFile = os.path.join(SAMPLECODE_TEMP_DIR, - 'samplecode_success.py') + if not os.path.exists(sampcd_processor.SAMPLECODE_TEMPDIR): + os.mkdir(sampcd_processor.SAMPLECODE_TEMPDIR) + self.successSampleCodeFile = os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_success.py') with open(self.successSampleCodeFile, 'w') as f: f.write('print(1+1)') - self.failedSampleCodeFile = os.path.join(SAMPLECODE_TEMP_DIR, - 'samplecode_failed.py') + self.failedSampleCodeFile = os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_failed.py') with open(self.failedSampleCodeFile, 'w') as f: f.write('print(1/0)') @@ -72,37 +294,41 @@ def tearDown(self): os.remove(self.failedSampleCodeFile) def test_run_success(self): - result, tfname, msg = execute_samplecode(self.successSampleCodeFile) + result, tfname, msg, exec_time = execute_samplecode( + self.successSampleCodeFile) self.assertTrue(result) self.assertEqual(self.successSampleCodeFile, tfname) self.assertIsNotNone(msg) self.assertLess(msg.find('skipped'), 0) + self.assertLess(exec_time, 10) def test_run_failed(self): - result, tfname, msg = execute_samplecode(self.failedSampleCodeFile) + result, tfname, msg, exec_time = execute_samplecode( + self.failedSampleCodeFile) self.assertFalse(result) self.assertEqual(self.failedSampleCodeFile, tfname) self.assertIsNotNone(msg) self.assertLess(msg.find('skipped'), 0) + self.assertLess(exec_time, 10) - def test_testcases_skipped(self): - ... - tfname = os.path.join(SAMPLECODE_TEMP_DIR, 'samplecode_skipped.py') - with open(tfname, 'w') as f: - f.write("# required: distributed\nprint(1/0)") - result, _, msg = execute_samplecode(tfname) - self.assertTrue(result) - self.assertGreaterEqual(msg.find('skipped'), 0) - os.remove(tfname) + +def clear_summary_info(): + for k in sampcd_processor.SUMMARY_INFO.keys(): + sampcd_processor.SUMMARY_INFO[k].clear() class Test_sampcd_extract_to_file(unittest.TestCase): def setUp(self): - if not os.path.exists(SAMPLECODE_TEMP_DIR): - os.mkdir(SAMPLECODE_TEMP_DIR) + if not os.path.exists(sampcd_processor.SAMPLECODE_TEMPDIR): + os.mkdir(sampcd_processor.SAMPLECODE_TEMPDIR) + clear_capacity() + os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu,distributed' + get_test_capacity() def tearDown(self): - shutil.rmtree(SAMPLECODE_TEMP_DIR) + shutil.rmtree(sampcd_processor.SAMPLECODE_TEMPDIR) + clear_capacity() + get_test_capacity() def test_1_samplecode(self): comments = """ @@ -113,9 +339,10 @@ def test_1_samplecode(self): """ funcname = 'one_plus_one' sample_code_filenames = sampcd_extract_to_file(comments, funcname) - self.assertCountEqual( - [os.path.join(SAMPLECODE_TEMP_DIR, funcname + '_example.py')], - sample_code_filenames) + self.assertCountEqual([ + os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example.py') + ], sample_code_filenames) def test_no_samplecode(self): comments = """ @@ -140,10 +367,64 @@ def test_2_samplecodes(self): funcname = 'one_plus_one' sample_code_filenames = sampcd_extract_to_file(comments, funcname) self.assertCountEqual([ - os.path.join(SAMPLECODE_TEMP_DIR, funcname + '_example_1.py'), - os.path.join(SAMPLECODE_TEMP_DIR, funcname + '_example_2.py') + os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_1.py'), + os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_2.py') ], sample_code_filenames) + def test_2_samplecodes_has_skipped(self): + comments = """ + placeholder + Examples: + .. code-block:: python + + # required: skiptest + print(1/0) + + .. code-block:: python + + print(1+1) + + .. code-block:: python + + # required: gpu + print(1//1) + + .. code-block:: python + + # required: xpu + print(1//1) + + .. code-block:: python + + # required: distributed + print(1//1) + + .. code-block:: python + + # required: gpu + print(1//1) + """ + funcname = 'one_plus_one' + clear_summary_info() + clear_capacity() + get_test_capacity() + + sample_code_filenames = sampcd_extract_to_file(comments, funcname) + self.assertCountEqual([ + os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_2.py') + ], sample_code_filenames) + self.assertCountEqual(sampcd_processor.SUMMARY_INFO['skiptest'], + [funcname + '-1']) + self.assertCountEqual(sampcd_processor.SUMMARY_INFO['gpu'], + [funcname + '-3', funcname + '-6']) + self.assertCountEqual(sampcd_processor.SUMMARY_INFO['xpu'], + [funcname + '-4']) + self.assertCountEqual(sampcd_processor.SUMMARY_INFO['distributed'], + [funcname + '-5']) + class Test_get_api_md5(unittest.TestCase): def setUp(self): @@ -208,55 +489,6 @@ def test_it(self): ], lines) -class Test_get_wlist(unittest.TestCase): - def setUp(self): - self.tmpDir = tempfile.mkdtemp() - self.wlist_filename = os.path.join(self.tmpDir, 'wlist.json') - with open(self.wlist_filename, 'w') as f: - f.write(r''' -{ - "wlist_dir":[ - { - "name":"../python/paddle/fluid/contrib", - "annotation":"" - }, - { - "name":"../python/paddle/verison.py", - "annotation":"" - } - ], - "wlist_api":[ - { - "name":"xxxxx", - "annotation":"not a real api, just for example" - } - ], - "wlist_temp_api":[ - "to_tensor", - "save_persistables@dygraph/checkpoint.py" - ], - "gpu_not_white":[ - "deformable_conv" - ] -} -''') - - def tearDown(self): - os.remove(self.wlist_filename) - shutil.rmtree(self.tmpDir) - - def test_get_wlist(self): - wlist, wlist_file, gpu_not_white = get_wlist(self.wlist_filename) - self.assertCountEqual( - ["xxxxx", "to_tensor", - "save_persistables@dygraph/checkpoint.py"], wlist) - self.assertCountEqual([ - "../python/paddle/fluid/contrib", - "../python/paddle/verison.py", - ], wlist_file) - self.assertCountEqual(["deformable_conv"], gpu_not_white) - - # https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/fluid/layers/ops.py # why? unabled to use the ast module. emmmmm diff --git a/tools/wlist.json b/tools/wlist.json deleted file mode 100644 index 5a83a9ee47004..0000000000000 --- a/tools/wlist.json +++ /dev/null @@ -1,505 +0,0 @@ -{ - "wlist_dir":[ - { - "name":"../python/paddle/fluid/contrib", - "annotation":"" - }, - { - "name":"../python/paddle/verison.py", - "annotation":"" - }, - { - "name":"../python/paddle/fluid/core_avx.py", - "annotation":"" - }, - { - "name":"../python/paddle/distributed", - "annotation":"" - } - ], - "wlist_api":[ - { - "name":"xxxxx", - "annotation":"not a real api, just for example" - }, - { - "name":"squeeze_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"unsqueeze_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"reshape_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"flatten_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"scatter_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"elu_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"relu_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"softmax_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"tanh_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"ceil_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"floor_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"exp_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"reciprocal_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"round_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"sqrt_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"rsqrt_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"clip_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"scale_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"subtract_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - }, - { - "name":"add_", - "annotation":"Inplace APIs don't need sample code. There is a special document introducing Inplace strategy" - } - ], - "wlist_temp_api":[ - "to_tensor", - "LRScheduler", - "ReduceOnPlateau", - "append_LARS", - "BuildStrategy.debug_graphviz_path", - "BuildStrategy.enable_sequential_execution", - "BuildStrategy.fuse_elewise_add_act_ops", - "BuildStrategy.fuse_relu_depthwise_conv", - "BuildStrategy.gradient_scale_strategy", - "BuildStrategy.reduce_strategy", - "BuildStrategy.remove_unnecessary_lock", - "BuildStrategy.sync_batch_norm", - "DynamicRNN.step_input", - "DynamicRNN.static_input", - "DynamicRNN.block", - "DynamicRNN.update_memory", - "DynamicRNN.output", - "transpiler.DistributeTranspilerConfig", - "transpiler.DistributeTranspilerConfig.slice_var_up", - "transpiler.DistributeTranspilerConfig.split_method", - "transpiler.DistributeTranspilerConfig.min_block_size", - "DistributeTranspilerConfig.slice_var_up", - "DistributeTranspilerConfig.split_method", - "ModelAverage.apply", - "ModelAverage.restore", - "DistributeTranspilerConfig", - "DistributeTranspilerConfig.min_block_size", - "ExecutionStrategy.allow_op_delay", - "load", - "Accuracy.update", - "ChunkEvaluator.update", - "ExecutionStrategy.num_iteration_per_drop_scope", - "ExecutionStrategy.num_threads", - "CompiledProgram._with_inference_optimize", - "CompositeMetric.add_metric", - "CompositeMetric.update", - "CompositeMetric.eval", - "DetectionMAP.get_map_var", - "MetricBase", - "MetricBase.reset", - "MetricBase.get_config", - "MetricBase.update", - "MetricBase.eval", - "Accuracy.eval", - "Auc.update", - "Auc.eval", - "EditDistance.update", - "EditDistance.eval", - "ExponentialMovingAverage.apply", - "ExponentialMovingAverage.restore", - "ExponentialMovingAverage.update", - "StaticRNN.step", - "StaticRNN.step_input", - "StaticRNN.step_output", - "StaticRNN.update_memory", - "DetectionMAP.reset", - "StaticRNN.output", - "cuda_places", - "CUDAPinnedPlace", - "CUDAPlace", - "Program.parse_from_string", - "Compressor", - "Compressor.config", - "Compressor.run", - "HDFSClient.upload", - "HDFSClient.download", - "HDFSClient.is_exist", - "HDFSClient.is_dir", - "HDFSClient.delete", - "HDFSClient.rename", - "HDFSClient.makedirs", - "HDFSClient.ls", - "HDFSClient.lsr", - "multi_download", - "multi_upload", - "TrainingDecoder.block", - "QuantizeTranspiler.training_transpile", - "QuantizeTranspiler.freeze_program", - "AutoMixedPrecisionLists", - "Uniform.sample", - "Uniform.log_prob", - "Uniform.entropy", - "Categorical.kl_divergence", - "Categorical.entropy", - "MultivariateNormalDiag.entropy", - "MultivariateNormalDiag.kl_divergence", - "RNNCell", - "RNNCell.call", - "RNNCell.get_initial_states", - "GRUCell.call", - "LSTMCell.call", - "Decoder", - "Decoder.initialize", - "Decoder.step", - "Decoder.finalize", - "fused_elemwise_activation", - "search_pyramid_hash", - "convert_dist_to_sparse_program", - "load_persistables_for_increment", - "load_persistables_for_inference", - "xmap_readers", - "Metric.reset", - "Metric.update", - "Metric.accumulate", - "Metric.name", - "Metric.compute", - "Accuracy.reset", - "Accuracy.update", - "Accuracy.accumulate", - "Accuracy.name", - "Accuracy.compute", - "Precision.reset", - "Precision.update", - "Precision.accumulate", - "Precision.name", - "Precision.compute", - "Recall.reset", - "Recall.update", - "Recall.accumulate", - "Recall.name", - "Recall.compute", - "Auc.reset", - "Auc.update", - "Auc.accumulate", - "Auc.name", - "Auc.compute", - "Callback.set_params", - "Callback.on_train_begin", - "Callback.on_train_end", - "Callback.on_eval_begin", - "Callback.on_eval_end", - "Callback.on_test_begin", - "Callback.on_test_end", - "Callback.on_epoch_begin", - "Callback.on_epoch_end", - "Callback.on_train_batch_begin", - "Callback.on_train_batch_end", - "Callback.on_eval_batch_begin", - "Callback.on_eval_batch_end", - "Callback.on_test_batch_begin", - "Callback.on_test_batch_end", - "Model.prepare", - "SimpleRNNCell", - "SimpleRNNCell.forward", - "LSTMCell", - "LSTMCell.forward", - "GRUCell", - "GRUCell.forward", - "SimpleRNN", - "GRU", - "LSTM", - "RNN", - "BiRNN", - "RNNCellBase", - "RNNCellBase.get_initial_states", - "gelu", - "erf", - "DecodeHelper", - "DecodeHelper.initialize", - "DecodeHelper.sample", - "DecodeHelper.next_inputs", - "TrainingHelper.initialize", - "TrainingHelper.sample", - "TrainingHelper.next_inputs", - "GreedyEmbeddingHelper.initialize", - "GreedyEmbeddingHelper.sample", - "GreedyEmbeddingHelper.next_inputs", - "LayerList.append", - "HDFSClient", - "InitState", - "TracedLayer", - "SampleEmbeddingHelper.sample", - "BasicDecoder.initialize", - "BasicDecoder.step", - "ParameterList.append", - "GreedyEmbeddingHelper", - "SampleEmbeddingHelper", - "BasicDecoder", - "lstm", - "partial_sum", - "StateCell", - "StateCell.compute_state", - "TrainingDecoder", - "TrainingDecoder.step_input", - "TrainingDecoder.static_input", - "TrainingDecoder.output", - "BeamSearchDecoder", - "GradClipByValue", - "GradClipByNorm", - "Variable.detach", - "Variable.numpy", - "Variable.set_value", - "Variable.gradient", - "BeamSearchDecoder.decode", - "BeamSearchDecoder.read_array", - "CompiledProgram", - "CompiledProgram.with_data_parallel", - "append_backward", - "guard", - "to_variable", - "op_freq_statistic", - "save_dygraph", - "load_dygraph", - "ParallelExecutor", - "ParallelExecutor.run", - "ParallelExecutor.drop_local_exe_scopes", - "GradClipByGlobalNorm", - "extend_with_decoupled_weight_decay", - "switch", - "Normal", - "memory_usage", - "decorate", - "PiecewiseDecay", - "InverseTimeDecay", - "PolynomialDecay", - "NoamDecay", - "start_profiler", - "profiler", - "tree_conv", - "multiclass_nms2", - "DataFeedDesc", - "Conv2D", - "Conv3D", - "Conv3DTranspose", - "Embedding", - "NCE", - "PRelu", - "BilinearTensorProduct", - "GroupNorm", - "SpectralNorm", - "TreeConv", - "prroi_pool", - "ChunkEvaluator", - "EditDistance", - "ErrorClipByValue", - "Program.clone", - "cuda_pinned_places", - "DataFeeder", - "elementwise_floordiv", - "Layer", - "Layer.create_parameter", - "Layer.create_variable", - "Layer.sublayers", - "Layer.add_parameter", - "Layer.add_sublayer", - "Layer.parameters", - "Tracer", - "Layer.full_name", - "InMemoryDataset", - "layer_norm", - "bipartite_match", - "double_buffer", - "cumsum", - "thresholded_relu", - "group_norm", - "random_crop", - "row_conv", - "hard_shrink", - "ssd_loss", - "retinanet_target_assign", - "InMemoryDataset.global_shuffle", - "InMemoryDataset.get_memory_data_size", - "DetectionMAP", - "hash", - "InMemoryDataset.set_queue_num", - "LayerNorm", - "Preprocessor", - "chunk_eval", - "GRUUnit", - "ExponentialMovingAverage", - "QueueDataset.global_shuffle", - "NumpyArrayInitializer", - "create_py_reader_by_data", - "InMemoryDataset.local_shuffle", - "InMemoryDataset.get_shuffle_data_size", - "size", - "edit_distance", - "nce", - "BilinearInitializer", - "NaturalExpDecay", - "noam_decay", - "retinanet_detection_output", - "Pool2D", - "PipelineOptimizer", - "generate_mask_labels", - "isfinite", - "InMemoryDataset.set_fleet_send_batch_size", - "cuda_profiler", - "unfold", - "Executor", - "InMemoryDataset.load_into_memory", - "ExponentialDecay", - "BatchNorm", - "deformable_conv", - "InMemoryDataset.preload_into_memory", - "py_reader", - "linear_lr_warmup", - "InMemoryDataset.wait_preload_done", - "CosineDecay", - "roi_perspective_transform", - "unique", - "ones_like", - "LambOptimizer", - "InMemoryDataset.release_memory", - "Conv2DTranspose", - "QueueDataset.local_shuffle", - "save_persistables@dygraph/checkpoint.py", - "load_persistables@dygraph/checkpoint.py", - "elementwise_pow", - "WeightedAverage.reset", - "ChunkEvaluator.eval", - "NCE.forward", - "elementwise_div", - "BilinearTensorProduct.forward", - "NoamDecay.step", - "elementwise_min", - "PiecewiseDecay.step", - "Conv3DTranspose.forward", - "elementwise_add", - "IfElse.output", - "IfElse.true_block", - "InverseTimeDecay.step", - "PolynomialDecay.step", - "Precision.eval", - "enabled", - "elementwise_max", - "stop_gperf_profiler", - "IfElse.false_block", - "WeightedAverage.add", - "Auc.trapezoid_area", - "elementwise_mul", - "GroupNorm.forward", - "SpectralNorm.forward", - "elementwise_sub", - "Switch.case", - "IfElse.input", - "prepare_context", - "PRelu.forward", - "Recall.update", - "start_gperf_profiler", - "TreeConv.forward", - "Conv2D.forward", - "Switch.default", - "elementwise_mod", - "Precision.update", - "WeightedAverage.eval", - "Conv3D.forward", - "Embedding.forward", - "Recall.eval", - "FC.forward", - "While.block", - "DGCMomentumOptimizer", - "ParallelEnv", - "spawn", - "init_parallel_env", - "DataParallel", - "DataParallel.scale_loss", - "DataParallel.apply_collective_grads", - "BasicLSTMCell.forward", - "BasicGRUCell.forward", - "RNN.forward", - "StackedRNNCell.forward", - "StackedLSTMCell.forward", - "LSTM.forward", - "BidirectionalRNN.forward", - "BidirectionalLSTM.forward", - "StackedGRUCell.forward", - "GRU.forward", - "BidirectionalGRU.forward", - "DynamicDecode.forward", - "Conv1dPoolLayer.forward", - "CNNEncoder.forward", - "TransformerCell.forward", - "TransformerBeamSearchDecoder.step", - "MultiHeadAttention.forward", - "MultiHeadAttention.cal_kv", - "FFN.forward", - "TransformerEncoderLayer.forward", - "TransformerEncoder.forward", - "TransformerDecoderLayer.forward", - "TransformerDecoder.forward", - "TransformerDecoder.prepare_static_cache", - "TransformerDecoder.prepare_incremental_cache", - "LinearChainCRF.forward", - "CRFDecoding.forward", - "SequenceTagging.forward", - "XPUPlace", - "is_compiled_with_xpu", - "xpu_places" - ], - "gpu_not_white":[ - "deformable_conv", - "cuda_places", - "CUDAPinnedPlace", - "CUDAPlace", - "cuda_profiler", - "DGCMomentumOptimizer" - ] -} From 14440905d5555e9903ee7b99475de3f4cdcc4348 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Fri, 11 Jun 2021 11:32:29 +0800 Subject: [PATCH 099/156] [Cherry-pick] Support diff dataset tensor place in single process dataloader (#33470) (#33487) Support diff dataset tensor place in single process dataloader cherry-pick of #33470 --- .../fluid/operators/reader/buffered_reader.cc | 18 ++++---- .../fluid/operators/reader/buffered_reader.h | 1 - .../unittests/test_dataloader_dataset.py | 46 +++++++++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/operators/reader/buffered_reader.cc b/paddle/fluid/operators/reader/buffered_reader.cc index f5d55791d86c6..17c84530b23e6 100644 --- a/paddle/fluid/operators/reader/buffered_reader.cc +++ b/paddle/fluid/operators/reader/buffered_reader.cc @@ -68,7 +68,6 @@ BufferedReader::BufferedReader( stream_ = platform::NpuStreamResourcePool::Instance().New(dev_idx); } #endif - is_same_place_ = false; cpu_buffer_.resize(buffer_size); cuda_buffer_.resize(buffer_size); npu_buffer_.resize(buffer_size); @@ -116,7 +115,7 @@ void BufferedReader::ReadAsync(size_t i) { std::vector cuda_pinned_ptrs; cuda_pinned_ptrs.reserve(cpu.size()); platform::RecordEvent record_event("BufferedReader:MemoryCopy"); - // NODE(chenwehiang): When we use CUDAPinned Memory, we need call + // NODE(chenweihang): When we use CUDAPinned Memory, we need call // cudaHostAlloc, that is a CUDA API, calling CUDA API need load // cuda lib into device, it will cost hundreds of MB of GPU memory. // If we don't set Device here, which will use CUDAPlace(0) default. @@ -126,18 +125,21 @@ void BufferedReader::ReadAsync(size_t i) { if (platform::is_cpu_place(cpu[i].place())) { cuda[i].Resize(cpu[i].dims()); cuda[i].set_layout(cpu[i].layout()); - cuda_pinned_ptrs.emplace_back( - cuda[i].mutable_data(cuda_pinned_place, cpu[i].type())); + cuda_pinned_ptrs[i] = + cuda[i].mutable_data(cuda_pinned_place, cpu[i].type()); auto size = cpu[i].numel() * paddle::framework::SizeOfType(cpu[i].type()); memory::Copy(cuda_pinned_place, cuda_pinned_ptrs[i], BOOST_GET_CONST(platform::CPUPlace, cpu[i].place()), cpu[i].data(), size); + cuda[i].set_lod(cpu[i].lod()); } else { - // we set same place flag & use cpu[i] directly - is_same_place_ = true; + // Here the cpu[i]'s place may be CUDAPlace, CUDAPinnedPlace, or + // others, we don't copy the memory of it to CUDAPinnedPlace, but + // we should share tensor data to cuda[i] + cuda[i].ShareDataWith(cpu[i]); } } } else { @@ -296,9 +298,9 @@ void BufferedReader::ReadNextImpl(std::vector *out) { return; } - if (platform::is_gpu_place(place_) && !is_same_place_) { + if (platform::is_gpu_place(place_)) { *out = std::move(cuda_buffer_[i]); - } else if (platform::is_npu_place(place_) && !is_same_place_) { + } else if (platform::is_npu_place(place_)) { *out = std::move(npu_buffer_[i]); } else { *out = std::move(cpu_buffer_[i]); diff --git a/paddle/fluid/operators/reader/buffered_reader.h b/paddle/fluid/operators/reader/buffered_reader.h index 9f7b0e753281e..5b4bbc7d62cd8 100644 --- a/paddle/fluid/operators/reader/buffered_reader.h +++ b/paddle/fluid/operators/reader/buffered_reader.h @@ -67,7 +67,6 @@ class BufferedReader : public framework::DecoratedReader { // buffer, just read async and create futures as buffer size. However, to // malloc tensors every time is extremely slow. Here we store all data in // buffers and prevent alloc every time. - bool is_same_place_; std::vector cpu_buffer_; std::vector cuda_buffer_; std::vector npu_buffer_; diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py index b8c498fe4a3c7..08589f0191d8c 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py @@ -14,9 +14,12 @@ from __future__ import division +import sys import unittest import numpy as np +import paddle +import paddle.vision.transforms as transforms import paddle.fluid as fluid from paddle.io import * @@ -37,5 +40,48 @@ def test_main(self): pass +class TestDatasetWithDiffOutputPlace(unittest.TestCase): + def get_dataloader(self, num_workers): + dataset = paddle.vision.datasets.MNIST( + mode='test', transform=transforms.ToTensor()) + loader = paddle.io.DataLoader( + dataset, batch_size=32, num_workers=num_workers, shuffle=True) + return loader + + def run_check_on_cpu(self): + paddle.set_device('cpu') + loader = self.get_dataloader(0) + for image, label in loader: + self.assertTrue(image.place.is_cpu_place()) + self.assertTrue(label.place.is_cpu_place()) + break + + def test_single_process(self): + self.run_check_on_cpu() + if paddle.is_compiled_with_cuda(): + # Get (image, label) tuple from MNIST dataset + # - the image is on CUDAPlace, label is on CPUPlace + paddle.set_device('gpu') + loader = self.get_dataloader(0) + for image, label in loader: + self.assertTrue(image.place.is_gpu_place()) + self.assertTrue(label.place.is_cuda_pinned_place()) + break + + def test_multi_process(self): + # DataLoader with multi-process mode is not supported on MacOs and Windows currently + if sys.platform != 'darwin' and sys.platform != 'win32': + self.run_check_on_cpu() + if paddle.is_compiled_with_cuda(): + # Get (image, label) tuple from MNIST dataset + # - the image and label are on CPUPlace + paddle.set_device('gpu') + loader = self.get_dataloader(1) + for image, label in loader: + self.assertTrue(image.place.is_cuda_pinned_place()) + self.assertTrue(label.place.is_cuda_pinned_place()) + break + + if __name__ == '__main__': unittest.main() From 9567cbd79681fa26acd283a6b30abdb4f080e53f Mon Sep 17 00:00:00 2001 From: liuyuhui Date: Fri, 11 Jun 2021 11:36:06 +0800 Subject: [PATCH 100/156] [cherry-pick 2.1.1]2.1/fix concat (#33383) * add unit8 for concat (#32850) * add bool type for tril api (#33402) --- paddle/fluid/operators/concat_op.cc | 6 ++++-- paddle/fluid/operators/concat_op.cu.cc | 6 ++++-- paddle/fluid/operators/reduce_ops/reduce_mean_op.cc | 5 ++++- paddle/fluid/operators/reduce_ops/reduce_mean_op.cu | 3 ++- paddle/fluid/operators/reduce_ops/reduce_mean_op.part.cu | 3 ++- paddle/fluid/operators/reduce_ops/reduce_sum_op.cc | 7 +++++-- paddle/fluid/operators/reduce_ops/reduce_sum_op.cu | 3 ++- paddle/fluid/operators/reduce_ops/reduce_sum_op.part.cu | 3 ++- paddle/fluid/operators/tril_triu_op.cc | 4 +++- paddle/fluid/operators/tril_triu_op.cu | 3 ++- python/paddle/tensor/creation.py | 2 +- python/paddle/tensor/manipulation.py | 2 +- 12 files changed, 32 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/operators/concat_op.cc b/paddle/fluid/operators/concat_op.cc index bbc42d97146f2..68a52a79e4ce3 100644 --- a/paddle/fluid/operators/concat_op.cc +++ b/paddle/fluid/operators/concat_op.cc @@ -233,7 +233,8 @@ REGISTER_OP_CPU_KERNEL( ops::ConcatKernel, ops::ConcatKernel, - ops::ConcatKernel); + ops::ConcatKernel, + ops::ConcatKernel); REGISTER_OP_CPU_KERNEL( concat_grad, ops::ConcatGradKernel, @@ -242,4 +243,5 @@ REGISTER_OP_CPU_KERNEL( ops::ConcatGradKernel, ops::ConcatGradKernel, - ops::ConcatGradKernel); + ops::ConcatGradKernel, + ops::ConcatKernel); diff --git a/paddle/fluid/operators/concat_op.cu.cc b/paddle/fluid/operators/concat_op.cu.cc index 8c30703f2576b..8732556acb9fd 100644 --- a/paddle/fluid/operators/concat_op.cu.cc +++ b/paddle/fluid/operators/concat_op.cu.cc @@ -23,7 +23,8 @@ REGISTER_OP_CUDA_KERNEL( ops::ConcatKernel, ops::ConcatKernel, ops::ConcatKernel, - ops::ConcatKernel); + ops::ConcatKernel, + ops::ConcatKernel); REGISTER_OP_CUDA_KERNEL( concat_grad, ops::ConcatGradKernel, @@ -31,4 +32,5 @@ REGISTER_OP_CUDA_KERNEL( ops::ConcatGradKernel, ops::ConcatGradKernel, ops::ConcatGradKernel, - ops::ConcatGradKernel); + ops::ConcatGradKernel, + ops::ConcatKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_mean_op.cc b/paddle/fluid/operators/reduce_ops/reduce_mean_op.cc index fdb2c57385b2b..c8d568c8c2cf7 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_mean_op.cc +++ b/paddle/fluid/operators/reduce_ops/reduce_mean_op.cc @@ -100,6 +100,8 @@ REGISTER_OPERATOR(reduce_mean_grad, ops::ReduceGradOp, ops::ReduceMeanDoubleGradOpBaseMaker, ops::ReduceMeanGradNoNeedBufferVarInferer); REGISTER_OP_CPU_KERNEL(reduce_mean, + ops::ReduceKernel, ops::ReduceKernel, ops::ReduceKernel; -REGISTER_OP_CPU_KERNEL(reduce_mean_grad, CPUReduceMeanGradKernel, +REGISTER_OP_CPU_KERNEL(reduce_mean_grad, CPUReduceMeanGradKernel, + CPUReduceMeanGradKernel, CPUReduceMeanGradKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu b/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu index cc3653fcb43a4..50d2fcdee23bd 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu +++ b/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu @@ -65,5 +65,6 @@ class ReduceMeanKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_CUDA_KERNEL(reduce_mean, ops::ReduceMeanKernel, +REGISTER_OP_CUDA_KERNEL(reduce_mean, ops::ReduceMeanKernel, + ops::ReduceMeanKernel, ops::ReduceMeanKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_mean_op.part.cu b/paddle/fluid/operators/reduce_ops/reduce_mean_op.part.cu index 289f574719ff0..0e133d5447f93 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_mean_op.part.cu +++ b/paddle/fluid/operators/reduce_ops/reduce_mean_op.part.cu @@ -20,5 +20,6 @@ using CUDAReduceMeanGradKernel = ops::ReduceGradKernel; -REGISTER_OP_CUDA_KERNEL(reduce_mean_grad, CUDAReduceMeanGradKernel, +REGISTER_OP_CUDA_KERNEL(reduce_mean_grad, CUDAReduceMeanGradKernel, + CUDAReduceMeanGradKernel, CUDAReduceMeanGradKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc b/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc index 5a8e8894e1c5d..a085e851eea77 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc +++ b/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc @@ -109,8 +109,10 @@ REGISTER_OPERATOR(reduce_sum_grad, ops::ReduceGradOp, ops::ReduceSumGradNoNeedBufferVarInferer); REGISTER_OP_CPU_KERNEL( - reduce_sum, ops::ReduceKernel, + ops::ReduceKernel, ops::ReduceKernel, ops::ReduceKernel, @@ -128,7 +130,8 @@ using CPUReduceSumGradKernel = ops::ReduceSumGradKernel; -REGISTER_OP_CPU_KERNEL(reduce_sum_grad, CPUReduceSumGradKernel, +REGISTER_OP_CPU_KERNEL(reduce_sum_grad, CPUReduceSumGradKernel, + CPUReduceSumGradKernel, CPUReduceSumGradKernel, CPUReduceSumGradKernel, CPUReduceSumGradKernel, diff --git a/paddle/fluid/operators/reduce_ops/reduce_sum_op.cu b/paddle/fluid/operators/reduce_ops/reduce_sum_op.cu index 219cc231a1ea7..dbd020514b208 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_sum_op.cu +++ b/paddle/fluid/operators/reduce_ops/reduce_sum_op.cu @@ -70,7 +70,8 @@ class ReduceSumKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_CUDA_KERNEL(reduce_sum, ops::ReduceSumKernel, +REGISTER_OP_CUDA_KERNEL(reduce_sum, ops::ReduceSumKernel, + ops::ReduceSumKernel, ops::ReduceSumKernel, ops::ReduceSumKernel, ops::ReduceSumKernel, ops::ReduceSumKernel, diff --git a/paddle/fluid/operators/reduce_ops/reduce_sum_op.part.cu b/paddle/fluid/operators/reduce_ops/reduce_sum_op.part.cu index f2bee6dddc39e..67de8bb9a0c1a 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_sum_op.part.cu +++ b/paddle/fluid/operators/reduce_ops/reduce_sum_op.part.cu @@ -20,7 +20,8 @@ using CUDAReduceSumGradKernel = ops::ReduceGradKernel; -REGISTER_OP_CUDA_KERNEL(reduce_sum_grad, CUDAReduceSumGradKernel, +REGISTER_OP_CUDA_KERNEL(reduce_sum_grad, CUDAReduceSumGradKernel, + CUDAReduceSumGradKernel, CUDAReduceSumGradKernel, CUDAReduceSumGradKernel, CUDAReduceSumGradKernel, diff --git a/paddle/fluid/operators/tril_triu_op.cc b/paddle/fluid/operators/tril_triu_op.cc index 8fb0b3809503e..3e943c62e1ce1 100644 --- a/paddle/fluid/operators/tril_triu_op.cc +++ b/paddle/fluid/operators/tril_triu_op.cc @@ -105,13 +105,15 @@ REGISTER_OPERATOR(tril_triu, ops::TrilTriuOp, ops::TrilTriuOpMaker, ops::TrilTriuGradOpMaker); REGISTER_OPERATOR(tril_triu_grad, ops::TrilTriuGradOp); REGISTER_OP_CPU_KERNEL( - tril_triu, ops::TrilTriuOpKernel, + tril_triu, ops::TrilTriuOpKernel, + ops::TrilTriuOpKernel, ops::TrilTriuOpKernel, ops::TrilTriuOpKernel, ops::TrilTriuOpKernel, ops::TrilTriuOpKernel); REGISTER_OP_CPU_KERNEL( tril_triu_grad, + ops::TrilTriuGradOpKernel, ops::TrilTriuGradOpKernel, ops::TrilTriuGradOpKernel, ops::TrilTriuGradOpKernel, diff --git a/paddle/fluid/operators/tril_triu_op.cu b/paddle/fluid/operators/tril_triu_op.cu index d04acd3405979..9cbbdeeb2ce28 100644 --- a/paddle/fluid/operators/tril_triu_op.cu +++ b/paddle/fluid/operators/tril_triu_op.cu @@ -18,7 +18,7 @@ namespace ops = paddle::operators; namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( - tril_triu, + tril_triu, ops::TrilTriuOpKernel, ops::TrilTriuOpKernel, ops::TrilTriuOpKernel, ops::TrilTriuOpKernel, @@ -26,6 +26,7 @@ REGISTER_OP_CUDA_KERNEL( ops::TrilTriuOpKernel); REGISTER_OP_CUDA_KERNEL( tril_triu_grad, + ops::TrilTriuGradOpKernel, ops::TrilTriuGradOpKernel, ops::TrilTriuGradOpKernel, ops::TrilTriuGradOpKernel, diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py index 361c0e80f90d7..5cede4369b278 100644 --- a/python/paddle/tensor/creation.py +++ b/python/paddle/tensor/creation.py @@ -576,7 +576,7 @@ def tril(x, diagonal=0, name=None): Args: x (Tensor): The input x which is a Tensor. - Support data types: ``float64``, ``float32``, ``int32``, ``int64``. + Support data types: ``bool``, ``float64``, ``float32``, ``int32``, ``int64``. diagonal (int, optional): The diagonal to consider, default value is 0. If :attr:`diagonal` = 0, all elements on and below the main diagonal are retained. A positive value includes just as many diagonals above the main diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 97826f7d5f81d..67e6c7f8e44d7 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -80,7 +80,7 @@ def concat(x, axis=0, name=None): Args: x(list|tuple): ``x`` is a Tensor list or Tensor tuple which is with data type bool, float16, - float32, float64, int32, int64. All the Tensors in ``x`` must have same data type. + float32, float64, int32, int64, uint8. All the Tensors in ``x`` must have same data type. axis(int|Tensor, optional): Specify the axis to operate on the input Tensors. It's a scalar with data type int or a Tensor with shape [1] and data type int32 or int64. The effective range is [-R, R), where R is Rank(x). When ``axis < 0``, From 45f8b9d08511af08394e99d9901a5ef5bb8201fe Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Fri, 11 Jun 2021 15:32:29 +0800 Subject: [PATCH 101/156] update 2.0 public api in vision (#33307) * update 2.0 public api in vision * fix some flake8 errors --- python/paddle/hapi/callbacks.py | 10 +-- python/paddle/hapi/model.py | 59 ++++++++------- python/paddle/metric/metrics.py | 2 +- python/paddle/tests/test_callback_visualdl.py | 2 +- python/paddle/vision/__init__.py | 63 ++++++++++++---- python/paddle/vision/datasets/__init__.py | 34 +++++---- python/paddle/vision/datasets/cifar.py | 2 +- python/paddle/vision/datasets/flowers.py | 2 +- python/paddle/vision/datasets/folder.py | 2 +- python/paddle/vision/datasets/mnist.py | 2 +- python/paddle/vision/datasets/voc2012.py | 2 +- python/paddle/vision/image.py | 2 +- python/paddle/vision/models/__init__.py | 50 +++++++++---- python/paddle/vision/models/lenet.py | 2 +- python/paddle/vision/models/mobilenetv1.py | 2 +- python/paddle/vision/models/mobilenetv2.py | 2 +- python/paddle/vision/models/resnet.py | 4 +- python/paddle/vision/models/vgg.py | 8 +- python/paddle/vision/ops.py | 8 +- python/paddle/vision/transforms/__init__.py | 73 +++++++++++++++++-- python/paddle/vision/transforms/functional.py | 6 +- .../vision/transforms/functional_cv2.py | 4 +- .../vision/transforms/functional_pil.py | 4 +- .../vision/transforms/functional_tensor.py | 2 + python/paddle/vision/transforms/transforms.py | 8 +- 25 files changed, 236 insertions(+), 119 deletions(-) diff --git a/python/paddle/hapi/callbacks.py b/python/paddle/hapi/callbacks.py index 61ae8b42d63a9..2bdde3879a2db 100644 --- a/python/paddle/hapi/callbacks.py +++ b/python/paddle/hapi/callbacks.py @@ -324,7 +324,7 @@ class ProgBarLogger(Callback): ]) train_dataset = MNIST(mode='train', transform=transform) - lenet = paddle.vision.LeNet() + lenet = paddle.vision.models.LeNet() model = paddle.Model(lenet, inputs, labels) @@ -554,7 +554,7 @@ class ModelCheckpoint(Callback): ]) train_dataset = MNIST(mode='train', transform=transform) - lenet = paddle.vision.LeNet() + lenet = paddle.vision.models.LeNet() model = paddle.Model(lenet, inputs, labels) @@ -614,7 +614,7 @@ class LRScheduler(Callback): ]) train_dataset = paddle.vision.datasets.MNIST(mode='train', transform=transform) - lenet = paddle.vision.LeNet() + lenet = paddle.vision.models.LeNet() model = paddle.Model(lenet, inputs, labels) @@ -630,7 +630,7 @@ def make_optimizer(parameters=None): boundaries=boundaries, values=values) learning_rate = paddle.optimizer.lr.LinearWarmup( learning_rate=learning_rate, - warmup_steps=wamup_epochs, + warmup_steps=wamup_steps, start_lr=base_lr / 5., end_lr=base_lr, verbose=True) @@ -856,7 +856,7 @@ class VisualDL(Callback): train_dataset = paddle.vision.datasets.MNIST(mode='train', transform=transform) eval_dataset = paddle.vision.datasets.MNIST(mode='test', transform=transform) - net = paddle.vision.LeNet() + net = paddle.vision.models.LeNet() model = paddle.Model(net, inputs, labels) optim = paddle.optimizer.Adam(0.001, parameters=net.parameters()) diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 160d6c54759d9..1c76c9174fd69 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -30,20 +30,28 @@ import paddle from paddle import fluid from paddle.fluid import core -from paddle.fluid.framework import in_dygraph_mode, Variable, ParamBase, _current_expected_place -from paddle.fluid.framework import in_dygraph_mode, Variable, _get_paddle_place +from paddle.fluid.framework import in_dygraph_mode +from paddle.fluid.framework import Variable +from paddle.fluid.framework import ParamBase +from paddle.fluid.framework import _current_expected_place +from paddle.fluid.framework import _get_paddle_place from paddle.fluid.framework import _current_expected_place as _get_device from paddle.fluid.executor import global_scope from paddle.fluid.io import is_belong_to_optimizer from paddle.fluid.dygraph.base import to_variable from paddle.fluid.dygraph.parallel import ParallelEnv -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator, FunctionSpec -from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import FunctionSpec +from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX +from paddle.fluid.dygraph.io import INFER_PARAMS_SUFFIX from paddle.fluid.layers.utils import flatten from paddle.fluid.layers import collective -from paddle.io import DataLoader, Dataset, DistributedBatchSampler -from paddle.fluid.executor import scope_guard, Executor +from paddle.io import DataLoader +from paddle.io import Dataset +from paddle.io import DistributedBatchSampler +from paddle.fluid.executor import scope_guard +from paddle.fluid.executor import Executor from paddle.fluid.dygraph.layers import Layer from paddle.metric import Metric from paddle.static import InputSpec as Input @@ -166,7 +174,6 @@ def init_communicator(program, rank, nranks, wait_port, current_endpoint, name=unique_name.generate('hccl_id'), persistable=True, type=core.VarDesc.VarType.RAW) - endpoint_to_index_map = {e: idx for idx, e in enumerate(endpoints)} block.append_op( type='c_gen_hccl_id', inputs={}, @@ -710,10 +717,10 @@ def train_batch(self, inputs, labels=None): enable=self._amp_level != 'O0', **self._amp_custom_lists): if self._nranks > 1: outputs = self.ddp_model.forward( - * [to_variable(x) for x in inputs]) + *[to_variable(x) for x in inputs]) else: outputs = self.model.network.forward( - * [to_variable(x) for x in inputs]) + *[to_variable(x) for x in inputs]) losses = self.model._loss(*(to_list(outputs) + labels)) losses = to_list(losses) @@ -732,7 +739,7 @@ def train_batch(self, inputs, labels=None): metrics = [] for metric in self.model._metrics: metric_outs = metric.compute(*(to_list(outputs) + labels)) - m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)]) + m = metric.update(*[to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) return ([to_numpy(l) for l in losses], metrics) \ @@ -746,7 +753,7 @@ def eval_batch(self, inputs, labels=None): labels = labels or [] labels = [to_variable(l) for l in to_list(labels)] - outputs = self.model.network.forward(* [to_variable(x) for x in inputs]) + outputs = self.model.network.forward(*[to_variable(x) for x in inputs]) if self.model._loss: losses = self.model._loss(*(to_list(outputs) + labels)) losses = to_list(losses) @@ -777,7 +784,7 @@ def eval_batch(self, inputs, labels=None): self._merge_count[self.mode + '_batch'] = samples metric_outs = metric.compute(*(to_list(outputs) + labels)) - m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)]) + m = metric.update(*[to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) if self.model._loss and len(metrics): @@ -1363,8 +1370,9 @@ def _check_pure_fp16_configs(): # pure float16 training has some restricts now if self._adapter._amp_level == "O2": if in_dygraph_mode(): - warnings.warn("Pure float16 training is not supported in dygraph mode now, "\ - "and it will be supported in future version.") + warnings.warn( + "Pure float16 training is not supported in dygraph mode now, and it will be supported in future version." + ) else: # grad clip is not supported in pure fp16 training now assert self._optimizer._grad_clip is None, \ @@ -1398,8 +1406,7 @@ def _check_pure_fp16_configs(): if 'use_pure_fp16' in amp_configs: raise ValueError( - "''use_pure_fp16' is an invalid parameter, " - "the level of mixed precision training only depends on 'O1' or 'O2'." + "'use_pure_fp16' is an invalid parameter, the level of mixed precision training only depends on 'O1' or 'O2'." ) _check_pure_fp16_configs() @@ -1427,9 +1434,8 @@ def _check_amp_configs(amp_config_key_set): } if amp_config_key_set - accepted_param_set: raise ValueError( - "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, " - "but {} could not be recognized.".format( - tuple(amp_config_key_set - accepted_param_set))) + "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but {} could not be recognized.". + format(tuple(amp_config_key_set - accepted_param_set))) if 'use_fp16_guard' in amp_config_key_set: if in_dygraph_mode(): @@ -1501,8 +1507,9 @@ def prepare(self, optimizer=None, loss=None, metrics=None, self._optimizer = optimizer if loss is not None: if not isinstance(loss, paddle.nn.Layer) and not callable(loss): - raise TypeError("'loss' must be sub classes of " \ - "`paddle.nn.Layer` or any callable function.") + raise TypeError( + "'loss' must be sub classes of `paddle.nn.Layer` or any callable function." + ) self._loss = loss metrics = metrics or [] @@ -2080,7 +2087,7 @@ def summary(self, input_size=None, dtype=None): input = InputSpec([None, 1, 28, 28], 'float32', 'image') label = InputSpec([None, 1], 'int64', 'label') - model = paddle.Model(paddle.vision.LeNet(), + model = paddle.Model(paddle.vision.models.LeNet(), input, label) optim = paddle.optimizer.Adam( learning_rate=0.001, parameters=model.parameters()) @@ -2122,9 +2129,11 @@ def _verify_spec(self, specs, shapes=None, dtypes=None, is_input=False): else: out_specs = to_list(specs) elif isinstance(specs, dict): - assert is_input == False - out_specs = [specs[n] \ - for n in extract_args(self.network.forward) if n != 'self'] + assert is_input is False + out_specs = [ + specs[n] for n in extract_args(self.network.forward) + if n != 'self' + ] else: out_specs = to_list(specs) # Note: checks each element has specificed `name`. diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 61d1eb0e37334..d8e400b08bd47 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -222,7 +222,7 @@ class Accuracy(Metric): transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) train_dataset = MNIST(mode='train', transform=transform) - model = paddle.Model(paddle.vision.LeNet(), input, label) + model = paddle.Model(paddle.vision.models.LeNet(), input, label) optim = paddle.optimizer.Adam( learning_rate=0.001, parameters=model.parameters()) model.prepare( diff --git a/python/paddle/tests/test_callback_visualdl.py b/python/paddle/tests/test_callback_visualdl.py index 36316183104fe..db3b83f2b1414 100644 --- a/python/paddle/tests/test_callback_visualdl.py +++ b/python/paddle/tests/test_callback_visualdl.py @@ -55,7 +55,7 @@ def test_visualdl_callback(self): train_dataset = MnistDataset(mode='train', transform=transform) eval_dataset = MnistDataset(mode='test', transform=transform) - net = paddle.vision.LeNet() + net = paddle.vision.models.LeNet() model = paddle.Model(net, inputs, labels) optim = paddle.optimizer.Adam(0.001, parameters=net.parameters()) diff --git a/python/paddle/vision/__init__.py b/python/paddle/vision/__init__.py index aeb07bf281fb0..79fb7844dd58c 100644 --- a/python/paddle/vision/__init__.py +++ b/python/paddle/vision/__init__.py @@ -11,22 +11,59 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import paddle +import paddle.nn as nn +from . import models # noqa: F401 +from . import transforms # noqa: F401 +from . import datasets # noqa: F401 +from . import ops # noqa: F401 +from .image import set_image_backend # noqa: F401 +from .image import get_image_backend # noqa: F401 +from .image import image_load # noqa: F401 +from .models import LeNet as models_LeNet +import paddle.utils.deprecated as deprecated -from . import models -from .models import * +__all__ = [ #noqa + 'set_image_backend', 'get_image_backend', 'image_load' +] -from . import transforms -from .transforms import * -from . import datasets -from .datasets import * +class LeNet(models_LeNet): + """LeNet model from + `"LeCun Y, Bottou L, Bengio Y, et al. Gradient-based learning applied to document recognition[J]. Proceedings of the IEEE, 1998, 86(11): 2278-2324.`_ -from . import image -from .image import * + Args: + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 10. -from . import ops + Examples: + .. code-block:: python -__all__ = models.__all__ \ - + transforms.__all__ \ - + datasets.__all__ \ - + image.__all__ + from paddle.vision.models import LeNet + + model = LeNet() + """ + + @deprecated( + since="2.0.0", + update_to="paddle.vision.models.LeNet", + level=1, + reason="Please use new API in models, paddle.vision.LeNet will be removed in future" + ) + def __init__(self, num_classes=10): + super(LeNet, self).__init__(num_classes=10) + self.num_classes = num_classes + self.features = nn.Sequential( + nn.Conv2D( + 1, 6, 3, stride=1, padding=1), + nn.ReLU(), + nn.MaxPool2D(2, 2), + nn.Conv2D( + 6, 16, 5, stride=1, padding=0), + nn.ReLU(), + nn.MaxPool2D(2, 2)) + + if num_classes > 0: + self.fc = nn.Sequential( + nn.Linear(400, 120), + nn.Linear(120, 84), nn.Linear(84, num_classes)) diff --git a/python/paddle/vision/datasets/__init__.py b/python/paddle/vision/datasets/__init__.py index 6703aa4197603..3ee7503e27979 100644 --- a/python/paddle/vision/datasets/__init__.py +++ b/python/paddle/vision/datasets/__init__.py @@ -12,20 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import folder -from . import mnist -from . import flowers -from . import cifar -from . import voc2012 +from .folder import DatasetFolder # noqa: F401 +from .folder import ImageFolder # noqa: F401 +from .mnist import MNIST # noqa: F401 +from .mnist import FashionMNIST # noqa: F401 +from .flowers import Flowers # noqa: F401 +from .cifar import Cifar10 # noqa: F401 +from .cifar import Cifar100 # noqa: F401 +from .voc2012 import VOC2012 # noqa: F401 -from .folder import * -from .mnist import * -from .flowers import * -from .cifar import * -from .voc2012 import * - -__all__ = folder.__all__ \ - + mnist.__all__ \ - + flowers.__all__ \ - + cifar.__all__ \ - + voc2012.__all__ +__all__ = [ #noqa + 'DatasetFolder' + 'ImageFolder', + 'MNIST', + 'FashionMNIST', + 'Flowers', + 'Cifar10', + 'Cifar100', + 'VOC2012' +] diff --git a/python/paddle/vision/datasets/cifar.py b/python/paddle/vision/datasets/cifar.py index 0a0a48026af80..2a582d7d0a8e5 100644 --- a/python/paddle/vision/datasets/cifar.py +++ b/python/paddle/vision/datasets/cifar.py @@ -24,7 +24,7 @@ from paddle.io import Dataset from paddle.dataset.common import _check_exists_and_download -__all__ = ['Cifar10', 'Cifar100'] +__all__ = [] URL_PREFIX = 'https://dataset.bj.bcebos.com/cifar/' CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz' diff --git a/python/paddle/vision/datasets/flowers.py b/python/paddle/vision/datasets/flowers.py index 448d6efb52bec..11b781b7a6dc7 100644 --- a/python/paddle/vision/datasets/flowers.py +++ b/python/paddle/vision/datasets/flowers.py @@ -25,7 +25,7 @@ from paddle.utils import try_import from paddle.dataset.common import _check_exists_and_download -__all__ = ["Flowers"] +__all__ = [] DATA_URL = 'http://paddlemodels.bj.bcebos.com/flowers/102flowers.tgz' LABEL_URL = 'http://paddlemodels.bj.bcebos.com/flowers/imagelabels.mat' diff --git a/python/paddle/vision/datasets/folder.py b/python/paddle/vision/datasets/folder.py index 718af041307a1..220b3d8ecb4b4 100644 --- a/python/paddle/vision/datasets/folder.py +++ b/python/paddle/vision/datasets/folder.py @@ -20,7 +20,7 @@ from paddle.io import Dataset from paddle.utils import try_import -__all__ = ["DatasetFolder", "ImageFolder"] +__all__ = [] def has_valid_extension(filename, extensions): diff --git a/python/paddle/vision/datasets/mnist.py b/python/paddle/vision/datasets/mnist.py index 1b998fd71a62e..84760f9598b6a 100644 --- a/python/paddle/vision/datasets/mnist.py +++ b/python/paddle/vision/datasets/mnist.py @@ -24,7 +24,7 @@ from paddle.io import Dataset from paddle.dataset.common import _check_exists_and_download -__all__ = ["MNIST", "FashionMNIST"] +__all__ = [] class MNIST(Dataset): diff --git a/python/paddle/vision/datasets/voc2012.py b/python/paddle/vision/datasets/voc2012.py index 1a42d143f0f72..5a82d7864cb00 100644 --- a/python/paddle/vision/datasets/voc2012.py +++ b/python/paddle/vision/datasets/voc2012.py @@ -23,7 +23,7 @@ from paddle.io import Dataset from paddle.dataset.common import _check_exists_and_download -__all__ = ["VOC2012"] +__all__ = [] VOC_URL = 'https://dataset.bj.bcebos.com/voc/VOCtrainval_11-May-2012.tar' diff --git a/python/paddle/vision/image.py b/python/paddle/vision/image.py index 19986816b7cc4..5c260b1d90a89 100644 --- a/python/paddle/vision/image.py +++ b/python/paddle/vision/image.py @@ -15,7 +15,7 @@ from PIL import Image from paddle.utils import try_import -__all__ = ['set_image_backend', 'get_image_backend', 'image_load'] +__all__ = [] _image_backend = 'pil' diff --git a/python/paddle/vision/models/__init__.py b/python/paddle/vision/models/__init__.py index 60d8c246ae10e..d38f3b1722ee8 100644 --- a/python/paddle/vision/models/__init__.py +++ b/python/paddle/vision/models/__init__.py @@ -12,20 +12,38 @@ #See the License for the specific language governing permissions and #limitations under the License. -from . import resnet -from . import vgg -from . import mobilenetv1 -from . import mobilenetv2 -from . import lenet +from .resnet import ResNet # noqa: F401 +from .resnet import resnet18 # noqa: F401 +from .resnet import resnet34 # noqa: F401 +from .resnet import resnet50 # noqa: F401 +from .resnet import resnet101 # noqa: F401 +from .resnet import resnet152 # noqa: F401 +from .mobilenetv1 import MobileNetV1 # noqa: F401 +from .mobilenetv1 import mobilenet_v1 # noqa: F401 +from .mobilenetv2 import MobileNetV2 # noqa: F401 +from .mobilenetv2 import mobilenet_v2 # noqa: F401 +from .vgg import VGG # noqa: F401 +from .vgg import vgg11 # noqa: F401 +from .vgg import vgg13 # noqa: F401 +from .vgg import vgg16 # noqa: F401 +from .vgg import vgg19 # noqa: F401 +from .lenet import LeNet # noqa: F401 -from .resnet import * -from .mobilenetv1 import * -from .mobilenetv2 import * -from .vgg import * -from .lenet import * - -__all__ = resnet.__all__ \ - + vgg.__all__ \ - + mobilenetv1.__all__ \ - + mobilenetv2.__all__ \ - + lenet.__all__ +__all__ = [ #noqa + 'ResNet', + 'resnet18', + 'resnet34', + 'resnet50', + 'resnet101', + 'resnet152', + 'VGG', + 'vgg11', + 'vgg13', + 'vgg16', + 'vgg19', + 'MobileNetV1', + 'mobilenet_v1', + 'MobileNetV2', + 'mobilenet_v2', + 'LeNet' +] diff --git a/python/paddle/vision/models/lenet.py b/python/paddle/vision/models/lenet.py index 2fb50fc17b9e9..46212f46f3a48 100644 --- a/python/paddle/vision/models/lenet.py +++ b/python/paddle/vision/models/lenet.py @@ -15,7 +15,7 @@ import paddle import paddle.nn as nn -__all__ = ['LeNet'] +__all__ = [] class LeNet(nn.Layer): diff --git a/python/paddle/vision/models/mobilenetv1.py b/python/paddle/vision/models/mobilenetv1.py index 22d177248e8b3..671a2cd8dfd5f 100644 --- a/python/paddle/vision/models/mobilenetv1.py +++ b/python/paddle/vision/models/mobilenetv1.py @@ -17,7 +17,7 @@ from paddle.utils.download import get_weights_path_from_url -__all__ = ['MobileNetV1', 'mobilenet_v1'] +__all__ = [] model_urls = { 'mobilenetv1_1.0': diff --git a/python/paddle/vision/models/mobilenetv2.py b/python/paddle/vision/models/mobilenetv2.py index f1cbaab1f90ac..74071fc121688 100644 --- a/python/paddle/vision/models/mobilenetv2.py +++ b/python/paddle/vision/models/mobilenetv2.py @@ -20,7 +20,7 @@ from paddle.utils.download import get_weights_path_from_url -__all__ = ['MobileNetV2', 'mobilenet_v2'] +__all__ = [] model_urls = { 'mobilenetv2_1.0': diff --git a/python/paddle/vision/models/resnet.py b/python/paddle/vision/models/resnet.py index 1f44e0bc6dfeb..5be69c93e8b5f 100644 --- a/python/paddle/vision/models/resnet.py +++ b/python/paddle/vision/models/resnet.py @@ -20,9 +20,7 @@ from paddle.utils.download import get_weights_path_from_url -__all__ = [ - 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152' -] +__all__ = [] model_urls = { 'resnet18': ('https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams', diff --git a/python/paddle/vision/models/vgg.py b/python/paddle/vision/models/vgg.py index f6b4c75e84f01..d526de8208329 100644 --- a/python/paddle/vision/models/vgg.py +++ b/python/paddle/vision/models/vgg.py @@ -17,13 +17,7 @@ from paddle.utils.download import get_weights_path_from_url -__all__ = [ - 'VGG', - 'vgg11', - 'vgg13', - 'vgg16', - 'vgg19', -] +__all__ = [] model_urls = { 'vgg16': ('https://paddle-hapi.bj.bcebos.com/models/vgg16.pdparams', diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index 60a7a90c9be89..53beedb885a71 100644 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -22,8 +22,12 @@ from paddle.common_ops_import import * -__all__ = [ - 'yolo_loss', 'yolo_box', 'deform_conv2d', 'DeformConv2D', 'read_file', +__all__ = [ #noqa + 'yolo_loss', + 'yolo_box', + 'deform_conv2d', + 'DeformConv2D', + 'read_file', 'decode_jpeg' ] diff --git a/python/paddle/vision/transforms/__init__.py b/python/paddle/vision/transforms/__init__.py index f7c5b63b19ed0..413f09f78699e 100644 --- a/python/paddle/vision/transforms/__init__.py +++ b/python/paddle/vision/transforms/__init__.py @@ -12,11 +12,70 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import transforms -from . import functional +from .transforms import BaseTransform # noqa: F401 +from .transforms import Compose # noqa: F401 +from .transforms import Resize # noqa: F401 +from .transforms import RandomResizedCrop # noqa: F401 +from .transforms import CenterCrop # noqa: F401 +from .transforms import RandomHorizontalFlip # noqa: F401 +from .transforms import RandomVerticalFlip # noqa: F401 +from .transforms import Transpose # noqa: F401 +from .transforms import Normalize # noqa: F401 +from .transforms import BrightnessTransform # noqa: F401 +from .transforms import SaturationTransform # noqa: F401 +from .transforms import ContrastTransform # noqa: F401 +from .transforms import HueTransform # noqa: F401 +from .transforms import ColorJitter # noqa: F401 +from .transforms import RandomCrop # noqa: F401 +from .transforms import Pad # noqa: F401 +from .transforms import RandomRotation # noqa: F401 +from .transforms import Grayscale # noqa: F401 +from .transforms import ToTensor # noqa: F401 +from .functional import to_tensor # noqa: F401 +from .functional import hflip # noqa: F401 +from .functional import vflip # noqa: F401 +from .functional import resize # noqa: F401 +from .functional import pad # noqa: F401 +from .functional import rotate # noqa: F401 +from .functional import to_grayscale # noqa: F401 +from .functional import crop # noqa: F401 +from .functional import center_crop # noqa: F401 +from .functional import adjust_brightness # noqa: F401 +from .functional import adjust_contrast # noqa: F401 +from .functional import adjust_hue # noqa: F401 +from .functional import normalize # noqa: F401 -from .transforms import * -from .functional import * - -__all__ = transforms.__all__ \ - + functional.__all__ +__all__ = [ #noqa + 'BaseTransform', + 'Compose', + 'Resize', + 'RandomResizedCrop', + 'CenterCrop', + 'RandomHorizontalFlip', + 'RandomVerticalFlip', + 'Transpose', + 'Normalize', + 'BrightnessTransform', + 'SaturationTransform', + 'ContrastTransform', + 'HueTransform', + 'ColorJitter', + 'RandomCrop', + 'Pad', + 'RandomRotation', + 'Grayscale', + 'ToTensor', + 'to_tensor', + 'hflip', + 'vflip', + 'resize', + 'pad', + 'rotate', + 'to_grayscale', + 'crop', + 'center_crop', + 'adjust_brightness', + 'adjust_contrast', + 'adjust_hue', + 'normalize' +] diff --git a/python/paddle/vision/transforms/functional.py b/python/paddle/vision/transforms/functional.py index 18a35915c99da..3087d5c3ed577 100644 --- a/python/paddle/vision/transforms/functional.py +++ b/python/paddle/vision/transforms/functional.py @@ -29,11 +29,7 @@ from . import functional_cv2 as F_cv2 from . import functional_tensor as F_t -__all__ = [ - 'to_tensor', 'hflip', 'vflip', 'resize', 'pad', 'rotate', 'to_grayscale', - 'crop', 'center_crop', 'adjust_brightness', 'adjust_contrast', 'adjust_hue', - 'normalize' -] +__all__ = [] def _is_pil_image(img): diff --git a/python/paddle/vision/transforms/functional_cv2.py b/python/paddle/vision/transforms/functional_cv2.py index 99cbfd6dc4f8d..487d79d276534 100644 --- a/python/paddle/vision/transforms/functional_cv2.py +++ b/python/paddle/vision/transforms/functional_cv2.py @@ -33,6 +33,8 @@ Sequence = collections.abc.Sequence Iterable = collections.abc.Iterable +__all__ = [] + def to_tensor(pic, data_format='CHW'): """Converts a ``numpy.ndarray`` to paddle.Tensor. @@ -49,7 +51,7 @@ def to_tensor(pic, data_format='CHW'): """ - if not data_format in ['CHW', 'HWC']: + if data_format not in ['CHW', 'HWC']: raise ValueError('data_format should be CHW or HWC. Got {}'.format( data_format)) diff --git a/python/paddle/vision/transforms/functional_pil.py b/python/paddle/vision/transforms/functional_pil.py index eee60c5452b2d..ae6d0cc45a92a 100644 --- a/python/paddle/vision/transforms/functional_pil.py +++ b/python/paddle/vision/transforms/functional_pil.py @@ -41,6 +41,8 @@ 'hamming': Image.HAMMING } +__all__ = [] + def to_tensor(pic, data_format='CHW'): """Converts a ``PIL.Image`` to paddle.Tensor. @@ -57,7 +59,7 @@ def to_tensor(pic, data_format='CHW'): """ - if not data_format in ['CHW', 'HWC']: + if data_format not in ['CHW', 'HWC']: raise ValueError('data_format should be CHW or HWC. Got {}'.format( data_format)) diff --git a/python/paddle/vision/transforms/functional_tensor.py b/python/paddle/vision/transforms/functional_tensor.py index 7f490d57916fb..1ec67416998a3 100644 --- a/python/paddle/vision/transforms/functional_tensor.py +++ b/python/paddle/vision/transforms/functional_tensor.py @@ -23,6 +23,8 @@ import sys import collections +__all__ = [] + def _assert_image_tensor(img, data_format): if not isinstance( diff --git a/python/paddle/vision/transforms/transforms.py b/python/paddle/vision/transforms/transforms.py index 00e12689c4d9f..8a35e6c3b908e 100644 --- a/python/paddle/vision/transforms/transforms.py +++ b/python/paddle/vision/transforms/transforms.py @@ -35,13 +35,7 @@ Sequence = collections.abc.Sequence Iterable = collections.abc.Iterable -__all__ = [ - "BaseTransform", "Compose", "Resize", "RandomResizedCrop", "CenterCrop", - "RandomHorizontalFlip", "RandomVerticalFlip", "Transpose", "Normalize", - "BrightnessTransform", "SaturationTransform", "ContrastTransform", - "HueTransform", "ColorJitter", "RandomCrop", "Pad", "RandomRotation", - "Grayscale", "ToTensor" -] +__all__ = [] def _get_image_size(img): From e48f7a5b4601b80cb7962c8675d66e61c79cde04 Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Fri, 11 Jun 2021 15:32:59 +0800 Subject: [PATCH 102/156] update 2.0 public api in all left files (#33314) * update 2.0 public api in all left files * reverse device.py all list; fix some flake8 errors --- python/paddle/__init__.py | 24 +++-------- python/paddle/amp/__init__.py | 4 +- python/paddle/amp/auto_cast.py | 2 +- python/paddle/amp/grad_scaler.py | 2 +- python/paddle/autograd/__init__.py | 9 ++--- python/paddle/autograd/backward_mode.py | 2 +- python/paddle/autograd/py_layer.py | 2 +- python/paddle/batch.py | 8 ++-- python/paddle/compat.py | 11 +---- python/paddle/device.py | 36 +++++++---------- python/paddle/distributed/parallel.py | 5 +-- python/paddle/incubate/__init__.py | 13 +++--- python/paddle/incubate/checkpoint/__init__.py | 4 +- python/paddle/incubate/optimizer/__init__.py | 6 +-- python/paddle/incubate/optimizer/lookahead.py | 11 ++--- .../paddle/incubate/optimizer/modelaverage.py | 19 ++++----- python/paddle/inference/__init__.py | 25 +++++++++++- python/paddle/jit/__init__.py | 33 +++++++++------ python/paddle/jit/dy2static/__init__.py | 36 +++++++++++------ .../paddle/jit/dy2static/convert_call_func.py | 4 +- .../paddle/jit/dy2static/convert_operators.py | 40 ++++++++----------- .../jit/dy2static/variable_trans_func.py | 18 ++++----- python/paddle/metric/__init__.py | 17 ++++++-- python/paddle/metric/metrics.py | 2 +- python/paddle/nn/__init__.py | 3 +- python/paddle/nn/functional/__init__.py | 3 +- python/paddle/onnx/__init__.py | 3 +- python/paddle/onnx/export.py | 2 +- python/paddle/static/__init__.py | 12 +++++- python/paddle/static/nn/__init__.py | 1 - python/paddle/tensor/__init__.py | 4 -- 31 files changed, 186 insertions(+), 175 deletions(-) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 7bac330376c44..e4cca3d459c4c 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -21,8 +21,7 @@ import paddle from the source directory; please install paddlepaddle*.whl firstly.''' ) -import paddle.batch -batch = batch.batch +from .batch import batch # noqa: F401 from .fluid import monkey_patch_variable from .fluid.dygraph import monkey_patch_math_varbase monkey_patch_variable() @@ -135,7 +134,6 @@ from .tensor.manipulation import squeeze_ # noqa: F401 from .tensor.manipulation import stack # noqa: F401 from .tensor.manipulation import strided_slice # noqa: F401 -from .tensor.manipulation import transpose # noqa: F401 from .tensor.manipulation import unique # noqa: F401 from .tensor.manipulation import unsqueeze # noqa: F401 from .tensor.manipulation import unsqueeze_ # noqa: F401 @@ -191,7 +189,6 @@ from .tensor.math import multiply # noqa: F401 from .tensor.math import add # noqa: F401 from .tensor.math import subtract # noqa: F401 -from .tensor.math import atan # noqa: F401 from .tensor.math import logsumexp # noqa: F401 from .tensor.math import inverse # noqa: F401 from .tensor.math import log1p # noqa: F401 @@ -244,9 +241,8 @@ from .framework import load # noqa: F401 from .framework import DataParallel # noqa: F401 -from .framework import set_default_dtype #DEFINE_ALIAS -from .framework import get_default_dtype #DEFINE_ALIAS -from .framework import set_grad_enabled #DEFINE_ALIAS +from .framework import set_default_dtype # noqa: F401 +from .framework import get_default_dtype # noqa: F401 from .tensor.search import index_sample # noqa: F401 from .tensor.stat import mean # noqa: F401 @@ -281,7 +277,7 @@ from .tensor.random import check_shape # noqa: F401 disable_static() -__all__ = [ #noqa +__all__ = [ # noqa 'dtype', 'uint8', 'int8', @@ -323,7 +319,6 @@ 'cos', 'tan', 'mean', - 'XPUPlace', 'mv', 'in_dynamic_mode', 'min', @@ -360,7 +355,6 @@ 'to_tensor', 'gather_nd', 'isinf', - 'set_device', 'uniform', 'floor_divide', 'remainder', @@ -384,8 +378,6 @@ 'rand', 'less_equal', 'triu', - 'is_compiled_with_cuda', - 'is_compiled_with_rocm', 'sin', 'dist', 'unbind', @@ -414,8 +406,6 @@ 'bernoulli', 'summary', 'sinh', - 'is_compiled_with_xpu', - 'is_compiled_with_npu', 'round', 'DataParallel', 'argmin', @@ -437,7 +427,6 @@ 'not_equal', 'sum', 'tile', - 'get_device', 'greater_equal', 'isfinite', 'create_parameter', @@ -470,7 +459,6 @@ 'scatter_nd', 'set_default_dtype', 'expand_as', - 'get_cudnn_version', 'stack', 'sqrt', 'cholesky', @@ -484,7 +472,6 @@ 'logical_not', 'add_n', 'minimum', - 'ComplexTensor', 'scatter', 'scatter_', 'floor', @@ -493,5 +480,6 @@ 'log2', 'log10', 'concat', - 'check_shape' + 'check_shape', + 'standard_normal' ] diff --git a/python/paddle/amp/__init__.py b/python/paddle/amp/__init__.py index 32587938512c4..64992752b2e8d 100644 --- a/python/paddle/amp/__init__.py +++ b/python/paddle/amp/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .auto_cast import auto_cast -from .grad_scaler import GradScaler +from .auto_cast import auto_cast # noqa: F401 +from .grad_scaler import GradScaler # noqa: F401 __all__ = ['auto_cast', 'GradScaler'] diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py index b83f81b27d1a0..974f718c2d4e2 100644 --- a/python/paddle/amp/auto_cast.py +++ b/python/paddle/amp/auto_cast.py @@ -14,7 +14,7 @@ from paddle.fluid.dygraph.amp import amp_guard -__all__ = ['auto_cast'] +__all__ = [] def auto_cast(enable=True, custom_white_list=None, custom_black_list=None): diff --git a/python/paddle/amp/grad_scaler.py b/python/paddle/amp/grad_scaler.py index 72a67a92c4958..770b660a9e11f 100644 --- a/python/paddle/amp/grad_scaler.py +++ b/python/paddle/amp/grad_scaler.py @@ -14,7 +14,7 @@ from paddle.fluid.dygraph.amp import AmpScaler -__all__ = ['GradScaler'] +__all__ = [] class GradScaler(AmpScaler): diff --git a/python/paddle/autograd/__init__.py b/python/paddle/autograd/__init__.py index 71110e9581787..569619f065a05 100644 --- a/python/paddle/autograd/__init__.py +++ b/python/paddle/autograd/__init__.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..fluid.dygraph.base import grad #DEFINE_ALIAS - -from . import backward_mode -from .backward_mode import backward -from .py_layer import PyLayer, PyLayerContext +from ..fluid.dygraph.base import grad # noqa: F401 +from . import backward_mode # noqa: F401 +from .backward_mode import backward # noqa: F401 +from .py_layer import PyLayer, PyLayerContext # noqa: F401 __all__ = ['grad', 'backward', 'PyLayer', 'PyLayerContext'] diff --git a/python/paddle/autograd/backward_mode.py b/python/paddle/autograd/backward_mode.py index 96e4336abaa6f..6efbe777d537c 100644 --- a/python/paddle/autograd/backward_mode.py +++ b/python/paddle/autograd/backward_mode.py @@ -15,7 +15,7 @@ from paddle.fluid import core from paddle.fluid import framework import paddle -__all__ = ['backward'] +__all__ = [] @framework.dygraph_only diff --git a/python/paddle/autograd/py_layer.py b/python/paddle/autograd/py_layer.py index 35e2cd2439177..5a22d22151a1c 100644 --- a/python/paddle/autograd/py_layer.py +++ b/python/paddle/autograd/py_layer.py @@ -15,7 +15,7 @@ import paddle from paddle.fluid.framework import dygraph_only from paddle.fluid import core -__all__ = ['PyLayer', 'PyLayerContext'] +__all__ = [] class PyLayerContext(object): diff --git a/python/paddle/batch.py b/python/paddle/batch.py index f6d2d8eb28874..f787f603f7e3a 100644 --- a/python/paddle/batch.py +++ b/python/paddle/batch.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['batch'] +__all__ = [] def batch(reader, batch_size, drop_last=False): @@ -35,11 +35,11 @@ def batch(reader, batch_size, drop_last=False): Examples: .. code-block:: python - import paddle.fluid as fluid + import paddle def reader(): for i in range(10): yield i - batch_reader = fluid.io.batch(reader, batch_size=2) + batch_reader = paddle.batch(reader, batch_size=2) for data in batch_reader(): print(data) @@ -60,7 +60,7 @@ def batch_reader(): if len(b) == batch_size: yield b b = [] - if drop_last == False and len(b) != 0: + if drop_last is False and len(b) != 0: yield b # Batch size check diff --git a/python/paddle/compat.py b/python/paddle/compat.py index 7c753815c5ccd..886a787623ed1 100644 --- a/python/paddle/compat.py +++ b/python/paddle/compat.py @@ -15,18 +15,11 @@ import six import math -__all__ = [ - 'long_type', - 'to_text', - 'to_bytes', - 'round', - 'floor_division', - 'get_exception_message', -] +__all__ = [] if six.PY2: int_type = int - long_type = long + long_type = long # noqa: F821 else: int_type = int long_type = int diff --git a/python/paddle/device.py b/python/paddle/device.py index 85b813a7f51b5..93e439ecf0aa4 100644 --- a/python/paddle/device.py +++ b/python/paddle/device.py @@ -18,21 +18,16 @@ from paddle.fluid import core from paddle.fluid import framework from paddle.fluid.dygraph.parallel import ParallelEnv -from paddle.fluid.framework import is_compiled_with_cuda #DEFINE_ALIAS -from paddle.fluid.framework import is_compiled_with_rocm #DEFINE_ALIAS +from paddle.fluid.framework import is_compiled_with_cuda # noqa: F401 +from paddle.fluid.framework import is_compiled_with_rocm # noqa: F401 -__all__ = [ + +__all__ = [ # npqa 'get_cudnn_version', 'set_device', 'get_device', 'XPUPlace', - 'is_compiled_with_xpu' - # 'cpu_places', - # 'CPUPlace', - # 'cuda_pinned_places', - # 'cuda_places', - # 'CUDAPinnedPlace', - # 'CUDAPlace', + 'is_compiled_with_xpu', 'is_compiled_with_cuda', 'is_compiled_with_rocm', 'is_compiled_with_npu' @@ -68,7 +63,7 @@ def is_compiled_with_xpu(): .. code-block:: python import paddle - support_xpu = paddle.device.is_compiled_with_xpu() + support_xpu = paddle.is_compiled_with_xpu() """ return core.is_compiled_with_xpu() @@ -82,9 +77,10 @@ def XPUPlace(dev_id): Examples: .. code-block:: python - + # required: xpu + import paddle - place = paddle.device.XPUPlace(0) + place = paddle.XPUPlace(0) """ return core.XPUPlace(dev_id) @@ -127,15 +123,13 @@ def _convert_to_place(device): place = core.CPUPlace() elif lower_device == 'gpu': if not core.is_compiled_with_cuda(): - raise ValueError( - "The device should not be 'gpu', " \ - "since PaddlePaddle is not compiled with CUDA") + raise ValueError("The device should not be 'gpu', " + "since PaddlePaddle is not compiled with CUDA") place = core.CUDAPlace(ParallelEnv().dev_id) elif lower_device == 'xpu': if not core.is_compiled_with_xpu(): - raise ValueError( - "The device should not be 'xpu', " \ - "since PaddlePaddle is not compiled with XPU") + raise ValueError("The device should not be 'xpu', " + "since PaddlePaddle is not compiled with XPU") selected_xpus = os.getenv("FLAGS_selected_xpus", "0").split(",") device_id = int(selected_xpus[0]) place = core.XPUPlace(device_id) @@ -149,7 +143,7 @@ def _convert_to_place(device): if avaliable_gpu_device: if not core.is_compiled_with_cuda(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ + "The device should not be {}, since PaddlePaddle is " "not compiled with CUDA".format(avaliable_gpu_device)) device_info_list = device.split(':', 1) device_id = device_info_list[1] @@ -158,7 +152,7 @@ def _convert_to_place(device): if avaliable_xpu_device: if not core.is_compiled_with_xpu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ + "The device should not be {}, since PaddlePaddle is " "not compiled with XPU".format(avaliable_xpu_device)) device_info_list = device.split(':', 1) device_id = device_info_list[1] diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py index bc042e722947a..efe747408428a 100644 --- a/python/paddle/distributed/parallel.py +++ b/python/paddle/distributed/parallel.py @@ -29,9 +29,7 @@ from paddle.fluid.dygraph.parallel import ParallelEnv from paddle.distributed.fleet.base.private_helper_function import wait_server_ready # noqa: F401 -__all__ = [ #noqa - "init_parallel_env" -] +__all__ = [] ParallelStrategy = core.ParallelStrategy @@ -152,7 +150,6 @@ def _check_var_exists(var_name): init_gloo = int(os.getenv("PADDLE_WITH_GLOO", "0")) if init_gloo: ep_rank_0 = parallel_env.trainer_endpoints[0].split(":") - ep_rank = parallel_env.trainer_endpoints[parallel_env.rank].split(":") manager = Manager() # glboal dict to store status http_server_d = manager.dict() diff --git a/python/paddle/incubate/__init__.py b/python/paddle/incubate/__init__.py index 03e5a88624086..22769053b1ac9 100644 --- a/python/paddle/incubate/__init__.py +++ b/python/paddle/incubate/__init__.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import optimizer -from . import checkpoint -from ..fluid.layer_helper import LayerHelper +from .optimizer import LookAhead # noqa: F401 +from .optimizer import ModelAverage # noqa: F401 +from .checkpoint import auto_checkpoint # noqa: F401 +from ..fluid.layer_helper import LayerHelper # noqa: F401 -__all__ = [] -__all__ += optimizer.__all__ -__all__ += checkpoint.__all__ +__all__ = [ # noqa + 'LookAhead', 'ModelAverage' +] diff --git a/python/paddle/incubate/checkpoint/__init__.py b/python/paddle/incubate/checkpoint/__init__.py index 7ddd256df7479..79e6259de0275 100644 --- a/python/paddle/incubate/checkpoint/__init__.py +++ b/python/paddle/incubate/checkpoint/__init__.py @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.incubate.checkpoint import auto_checkpoint +from ...fluid.incubate.checkpoint import auto_checkpoint # noqa: F401 -__all__ = ["auto_checkpoint"] +__all__ = [] diff --git a/python/paddle/incubate/optimizer/__init__.py b/python/paddle/incubate/optimizer/__init__.py index 4a3889d0ee1a9..d966d187f288a 100644 --- a/python/paddle/incubate/optimizer/__init__.py +++ b/python/paddle/incubate/optimizer/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .lookahead import LookAhead -from .modelaverage import ModelAverage +from .lookahead import LookAhead # noqa: F401 +from .modelaverage import ModelAverage # noqa: F401 -__all__ = ['LookAhead', 'ModelAverage'] +__all__ = [] diff --git a/python/paddle/incubate/optimizer/lookahead.py b/python/paddle/incubate/optimizer/lookahead.py index f90d520a5dfe8..720a84a24f0aa 100644 --- a/python/paddle/incubate/optimizer/lookahead.py +++ b/python/paddle/incubate/optimizer/lookahead.py @@ -20,7 +20,7 @@ import numpy as np from paddle.fluid.dygraph import base as imperative_base -__all__ = ["LookAhead"] +__all__ = [] class LookAhead(Optimizer): @@ -99,7 +99,7 @@ def train(layer, loader, loss_fn, opt): layer = LinearNet() loss_fn = nn.CrossEntropyLoss() optimizer = paddle.optimizer.SGD(learning_rate=0.1, parameters=layer.parameters()) - lookahead = paddle.incubate.optimizer.LookAhead(optimizer, alpha=0.2, k=5) + lookahead = paddle.incubate.LookAhead(optimizer, alpha=0.2, k=5) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) @@ -163,7 +163,7 @@ def step(self): out = linear(inp) loss = paddle.mean(out) sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters()) - lookahead = paddle.incubate.optimizer.LookAhead(sgd, alpha=0.2, k=5) + lookahead = paddle.incubate.LookAhead(sgd, alpha=0.2, k=5) loss.backward() lookahead.step() lookahead.clear_grad() @@ -274,7 +274,7 @@ def minimize(self, out = linear(inp) loss = paddle.mean(out) sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters()) - lookahead = paddle.incubate.optimizer.LookAhead(sgd, alpha=0.2, k=5) + lookahead = paddle.incubate.LookAhead(sgd, alpha=0.2, k=5) loss.backward() lookahead.minimize(loss) lookahead.clear_grad() @@ -282,9 +282,6 @@ def minimize(self, """ assert isinstance(loss, Variable), "The loss should be an Tensor." - parameter_list = parameters if parameters \ - else self._parameter_list - # Apply inner optimizer to the main_program optimize_ops, params_grads = self.inner_optimizer.minimize( loss, diff --git a/python/paddle/incubate/optimizer/modelaverage.py b/python/paddle/incubate/optimizer/modelaverage.py index 8afcaf9207e7c..8ffc3bdac62d0 100644 --- a/python/paddle/incubate/optimizer/modelaverage.py +++ b/python/paddle/incubate/optimizer/modelaverage.py @@ -21,7 +21,7 @@ from paddle.fluid.dygraph import base as imperative_base from paddle.fluid.wrapped_decorator import signature_safe_contextmanager -__all__ = ["ModelAverage"] +__all__ = [] class ModelAverage(Optimizer): @@ -129,7 +129,7 @@ def evaluate(layer, loader, loss_fn): layer = LinearNet() loss_fn = nn.CrossEntropyLoss() optimizer = opt.Momentum(learning_rate=0.2, momentum=0.1, parameters=layer.parameters()) - model_average = paddle.incubate.optimizer.ModelAverage(0.15, + model_average = paddle.incubate.ModelAverage(0.15, parameters=layer.parameters(), min_average_window=2, max_average_window=10) @@ -313,7 +313,7 @@ def minimize(self, sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters()) sgd.minimize(loss) - modelaverage = paddle.incubate.optimizer.ModelAverage(0.15, + modelaverage = paddle.incubate.ModelAverage(0.15, parameters=linear.parameters(), min_average_window=2, max_average_window=4) @@ -345,7 +345,7 @@ def step(self): out = linear(inp) loss = paddle.mean(out) sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters()) - modelaverage = paddle.incubate.optimizer.ModelAverage(0.15, + modelaverage = paddle.incubate.ModelAverage(0.15, parameters=linear.parameters(), min_average_window=2, max_average_window=4) @@ -395,7 +395,7 @@ def apply(self, executor=None, need_restore=True): sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters()) - modelaverage = paddle.incubate.optimizer.ModelAverage(0.15, + modelaverage = paddle.incubate.ModelAverage(0.15, parameters=linear.parameters(), min_average_window=2, max_average_window=4) @@ -415,7 +415,6 @@ def apply(self, executor=None, need_restore=True): param) old_num_accumulates = self._get_accumulator( 'old_num_accumulates', param) - num_updates = self._get_accumulator('num_updates', param) sum_1 = self._get_accumulator('sum_1', param) sum_2 = self._get_accumulator('sum_2', param) sum_3 = self._get_accumulator('sum_3', param) @@ -467,7 +466,7 @@ def restore(self, executor=None): sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters()) - modelaverage = paddle.incubate.optimizer.ModelAverage(0.15, + modelaverage = paddle.incubate.ModelAverage(0.15, parameters=linear.parameters(), min_average_window=2, max_average_window=4) @@ -506,17 +505,15 @@ def _add_average_apply_op(self, block, param): self._get_accumulator('num_accumulates', param)) old_num_accumulates = block._clone_variable( self._get_accumulator('old_num_accumulates', param)) - num_updates = block._clone_variable( - self._get_accumulator('num_updates', param)) # backup param value to grad layers.assign(input=param, output=grad) # param = (sum_1 + sum_2 + sum_3) / (num_accumulates + old_num_accumulates) tmp = layers.sum(x=[num_accumulates, old_num_accumulates]) sum = layers.sum(x=[sum_1, sum_2, sum_3]) tmp = layers.cast( - x=tmp, dtype='float32' if self._dtype == None else self._dtype) + x=tmp, dtype='float32' if self._dtype is None else self._dtype) sum = layers.cast( - x=sum, dtype='float32' if self._dtype == None else self._dtype) + x=sum, dtype='float32' if self._dtype is None else self._dtype) layers.ops._elementwise_div(x=sum, y=tmp, out=param) def _add_average_restore_op(self, block, param): diff --git a/python/paddle/inference/__init__.py b/python/paddle/inference/__init__.py index c388301ec3408..4e17203971662 100644 --- a/python/paddle/inference/__init__.py +++ b/python/paddle/inference/__init__.py @@ -12,5 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..fluid.inference import Config, DataType, PlaceType, PrecisionType, Tensor, \ - Predictor, create_predictor, get_version, get_num_bytes_of_data_type, PredictorPool +from ..fluid.inference import Config # noqa: F401 +from ..fluid.inference import DataType # noqa: F401 +from ..fluid.inference import PlaceType # noqa: F401 +from ..fluid.inference import PrecisionType # noqa: F401 +from ..fluid.inference import Tensor # noqa: F401 +from ..fluid.inference import Predictor # noqa: F401 +from ..fluid.inference import create_predictor # noqa: F401 +from ..fluid.inference import get_version # noqa: F401 +from ..fluid.inference import get_num_bytes_of_data_type # noqa: F401 +from ..fluid.inference import PredictorPool # noqa: F401 + +__all__ = [ # noqa + 'Config', + 'DataType', + 'PlaceType', + 'PrecisionType', + 'Tensor', + 'Predictor', + 'create_predictor', + 'get_version', + 'get_num_bytes_of_data_type', + 'PredictorPool' +] diff --git a/python/paddle/jit/__init__.py b/python/paddle/jit/__init__.py index 650837b2d7702..576989e8e0d2a 100644 --- a/python/paddle/jit/__init__.py +++ b/python/paddle/jit/__init__.py @@ -14,19 +14,26 @@ from __future__ import print_function -from ..fluid.dygraph.jit import save #DEFINE_ALIAS -from ..fluid.dygraph.jit import load #DEFINE_ALIAS -from ..fluid.dygraph.jit import TracedLayer #DEFINE_ALIAS -from ..fluid.dygraph.jit import set_code_level #DEFINE_ALIAS -from ..fluid.dygraph.jit import set_verbosity #DEFINE_ALIAS -from ..fluid.dygraph.jit import declarative as to_static #DEFINE_ALIAS -from ..fluid.dygraph.jit import not_to_static #DEFINE_ALIAS -from ..fluid.dygraph import ProgramTranslator #DEFINE_ALIAS -from ..fluid.dygraph.io import TranslatedLayer #DEFINE_ALIAS +from ..fluid.dygraph.jit import save # noqa: F401 +from ..fluid.dygraph.jit import load # noqa: F401 +from ..fluid.dygraph.jit import TracedLayer # noqa: F401 +from ..fluid.dygraph.jit import set_code_level # noqa: F401 +from ..fluid.dygraph.jit import set_verbosity # noqa: F401 +from ..fluid.dygraph.jit import declarative as to_static # noqa: F401 +from ..fluid.dygraph.jit import not_to_static # noqa: F401 +from ..fluid.dygraph import ProgramTranslator # noqa: F401 +from ..fluid.dygraph.io import TranslatedLayer # noqa: F401 -from . import dy2static +from . import dy2static # noqa: F401 -__all__ = [ - 'save', 'load', 'TracedLayer', 'to_static', 'ProgramTranslator', - 'TranslatedLayer', 'set_code_level', 'set_verbosity', 'not_to_static' +__all__ = [ # noqa + 'save', + 'load', + 'TracedLayer', + 'to_static', + 'ProgramTranslator', + 'TranslatedLayer', + 'set_code_level', + 'set_verbosity', + 'not_to_static' ] diff --git a/python/paddle/jit/dy2static/__init__.py b/python/paddle/jit/dy2static/__init__.py index 239b554180b1b..030d5499c2ca9 100644 --- a/python/paddle/jit/dy2static/__init__.py +++ b/python/paddle/jit/dy2static/__init__.py @@ -12,18 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -from . import convert_operators -from .convert_operators import * - -from . import convert_call_func -from .convert_call_func import * - -from . import variable_trans_func -from .variable_trans_func import * +from .convert_call_func import convert_call # noqa: F401 +from .convert_operators import cast_bool_if_necessary # noqa: F401 +from .convert_operators import convert_assert # noqa: F401 +from .convert_operators import convert_ifelse # noqa: F401 +from .convert_operators import convert_len # noqa: F401 +from .convert_operators import convert_logical_and # noqa: F401 +from .convert_operators import convert_logical_not # noqa: F401 +from .convert_operators import convert_logical_or # noqa: F401 +from .convert_operators import convert_pop # noqa: F401 +from .convert_operators import convert_print # noqa: F401 +from .convert_operators import convert_shape_compare # noqa: F401 +from .convert_operators import convert_var_dtype # noqa: F401 +from .convert_operators import convert_var_shape # noqa: F401 +from .convert_operators import convert_var_shape_simple # noqa: F401 +from .convert_operators import eval_if_exist_else_none # noqa: F401 +from .convert_operators import choose_shape_attr_or_api # noqa: F401 +from .convert_operators import convert_while_loop # noqa: F401 +from .variable_trans_func import create_bool_as_type # noqa: F401 +from .variable_trans_func import create_fill_constant_node # noqa: F401 +from .variable_trans_func import create_static_variable_gast_node # noqa: F401 +from .variable_trans_func import data_layer_not_check # noqa: F401 +from .variable_trans_func import to_static_variable # noqa: F401 +from .variable_trans_func import to_static_variable_gast_node # noqa: F401 __all__ = [] -__all__ += convert_operators.__all__ -__all__ += convert_call_func.__all__ -__all__ += variable_trans_func.__all__ diff --git a/python/paddle/jit/dy2static/convert_call_func.py b/python/paddle/jit/dy2static/convert_call_func.py index be2377608e36c..4f6197a3cba6a 100644 --- a/python/paddle/jit/dy2static/convert_call_func.py +++ b/python/paddle/jit/dy2static/convert_call_func.py @@ -13,6 +13,6 @@ # limitations under the License. from __future__ import print_function -from ...fluid.dygraph.dygraph_to_static.convert_call_func import convert_call #DEFINE_ALIAS +from ...fluid.dygraph.dygraph_to_static.convert_call_func import convert_call # noqa: F401 -__all__ = ['convert_call'] +__all__ = [] diff --git a/python/paddle/jit/dy2static/convert_operators.py b/python/paddle/jit/dy2static/convert_operators.py index 9321cf4a0b832..8d67e06d9b27a 100644 --- a/python/paddle/jit/dy2static/convert_operators.py +++ b/python/paddle/jit/dy2static/convert_operators.py @@ -13,27 +13,21 @@ # limitations under the License. from __future__ import print_function -from ...fluid.dygraph.dygraph_to_static.convert_operators import cast_bool_if_necessary #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_assert #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_ifelse #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_len #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_and #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_not #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_or #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_pop #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_print #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_shape_compare #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_dtype #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_shape #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_shape_simple #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import eval_if_exist_else_none #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import choose_shape_attr_or_api #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_while_loop #DEFINE_ALIAS +from ...fluid.dygraph.dygraph_to_static.convert_operators import cast_bool_if_necessary # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_assert # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_ifelse # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_len # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_and # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_not # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_or # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_pop # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_print # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_shape_compare # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_dtype # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_shape # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_shape_simple # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import eval_if_exist_else_none # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import choose_shape_attr_or_api # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_while_loop # noqa: F401 -__all__ = [ - 'cast_bool_if_necessary', 'convert_assert', 'convert_ifelse', 'convert_len', - 'convert_logical_and', 'convert_logical_not', 'convert_logical_or', - 'convert_pop', 'convert_print', 'convert_shape_compare', - 'convert_var_dtype', 'convert_var_shape', 'convert_var_shape_simple', - 'eval_if_exist_else_none', 'choose_shape_attr_or_api', 'convert_while_loop' -] +__all__ = [] diff --git a/python/paddle/jit/dy2static/variable_trans_func.py b/python/paddle/jit/dy2static/variable_trans_func.py index 2deb1bbb0eef2..9ce2bc2da3816 100644 --- a/python/paddle/jit/dy2static/variable_trans_func.py +++ b/python/paddle/jit/dy2static/variable_trans_func.py @@ -14,15 +14,11 @@ from __future__ import print_function -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_bool_as_type #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_static_variable_gast_node #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import data_layer_not_check #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable #DEFINE_ALIAS -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable_gast_node #DEFINE_ALIAS +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_bool_as_type # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_static_variable_gast_node # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import data_layer_not_check # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable_gast_node # noqa: F401 -__all__ = [ - 'create_bool_as_type', 'create_fill_constant_node', - 'create_static_variable_gast_node', 'data_layer_not_check', - 'to_static_variable', 'to_static_variable_gast_node' -] +__all__ = [] diff --git a/python/paddle/metric/__init__.py b/python/paddle/metric/__init__.py index e41f6d76dd221..2f2ef4c6f5426 100644 --- a/python/paddle/metric/__init__.py +++ b/python/paddle/metric/__init__.py @@ -12,7 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .metrics import * -from . import metrics +from .metrics import Metric # noqa: F401 +from .metrics import Accuracy # noqa: F401 +from .metrics import Precision # noqa: F401 +from .metrics import Recall # noqa: F401 +from .metrics import Auc # noqa: F401 +from .metrics import accuracy # noqa: F401 -__all__ = metrics.__all__ +__all__ = [ #noqa + 'Metric', + 'Accuracy', + 'Precision', + 'Recall', + 'Auc', + 'accuracy' +] diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index d8e400b08bd47..40758fb8dc3e0 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -26,7 +26,7 @@ from ..fluid.framework import core, _varbase_creator, in_dygraph_mode import paddle -__all__ = ['Metric', 'Accuracy', 'Precision', 'Recall', 'Auc', 'accuracy'] +__all__ = [] def _is_numpy_(var): diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 7cf3f94872de1..3ccb9e957f4e4 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -286,5 +286,6 @@ def weight_norm(*args): 'Swish', 'PixelShuffle', 'ELU', - 'ReLU6' + 'ReLU6', + 'LayerDict' ] diff --git a/python/paddle/nn/functional/__init__.py b/python/paddle/nn/functional/__init__.py index d4c17a27a6178..ff18afa9d2028 100644 --- a/python/paddle/nn/functional/__init__.py +++ b/python/paddle/nn/functional/__init__.py @@ -194,5 +194,6 @@ 'embedding', 'gather_tree', 'one_hot', - 'normalize' + 'normalize', + 'temporal_shift' ] diff --git a/python/paddle/onnx/__init__.py b/python/paddle/onnx/__init__.py index 885d1968ce1ae..8853e78bf3d80 100644 --- a/python/paddle/onnx/__init__.py +++ b/python/paddle/onnx/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function -from .export import export +from .export import export # noqa: F401 __all__ = ['export'] diff --git a/python/paddle/onnx/export.py b/python/paddle/onnx/export.py index 4b99b42bb0423..b8a217a5134fb 100644 --- a/python/paddle/onnx/export.py +++ b/python/paddle/onnx/export.py @@ -15,7 +15,7 @@ import os from paddle.utils import try_import -__all__ = ['export'] +__all__ = [] def export(layer, path, input_spec=None, opset_version=9, **configs): diff --git a/python/paddle/static/__init__.py b/python/paddle/static/__init__.py index 688bff4a678f2..93394f9b5afde 100644 --- a/python/paddle/static/__init__.py +++ b/python/paddle/static/__init__.py @@ -85,11 +85,21 @@ 'load', 'save_inference_model', 'load_inference_model', + 'serialize_program', + 'serialize_persistables', + 'save_to_file', + 'deserialize_program', + 'deserialize_persistables', + 'load_from_file', 'normalize_program', 'load_program_state', 'set_program_state', 'cpu_places', 'cuda_places', 'Variable', - 'create_global_var' + 'create_global_var', + 'accuracy', + 'auc', + 'device_guard', + 'create_parameter' ] diff --git a/python/paddle/static/nn/__init__.py b/python/paddle/static/nn/__init__.py index 416f6e4f3df06..b589d9f87895b 100644 --- a/python/paddle/static/nn/__init__.py +++ b/python/paddle/static/nn/__init__.py @@ -68,7 +68,6 @@ 'conv2d_transpose', 'conv3d', 'conv3d_transpose', - 'create_parameter', 'crf_decoding', 'data_norm', 'deform_conv2d', diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py index c8d80fc9bc68c..5aeae126d8376 100755 --- a/python/paddle/tensor/__init__.py +++ b/python/paddle/tensor/__init__.py @@ -25,7 +25,6 @@ from .creation import zeros # noqa: F401 from .creation import zeros_like # noqa: F401 from .creation import arange # noqa: F401 -from .creation import eye # noqa: F401 from .creation import full # noqa: F401 from .creation import full_like # noqa: F401 from .creation import triu # noqa: F401 @@ -82,7 +81,6 @@ from .manipulation import squeeze_ # noqa: F401 from .manipulation import stack # noqa: F401 from .manipulation import strided_slice # noqa: F401 -from .manipulation import transpose # noqa: F401 from .manipulation import unique # noqa: F401 from .manipulation import unsqueeze # noqa: F401 from .manipulation import unsqueeze_ # noqa: F401 @@ -143,7 +141,6 @@ from .math import add_ # noqa: F401 from .math import subtract # noqa: F401 from .math import subtract_ # noqa: F401 -from .math import atan # noqa: F401 from .math import logsumexp # noqa: F401 from .math import inverse # noqa: F401 from .math import log2 # noqa: F401 @@ -227,7 +224,6 @@ 'log2', 'log10', 'logsumexp', - 'mul', 'multiplex', 'pow', 'prod', From de612f76261e85a614660ddeabb575f7fcb018bd Mon Sep 17 00:00:00 2001 From: zhoujun Date: Fri, 11 Jun 2021 04:05:45 -0500 Subject: [PATCH 103/156] Add comments to ColorJitter parameters;test=document_fix (#33432) --- python/paddle/vision/transforms/transforms.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/vision/transforms/transforms.py b/python/paddle/vision/transforms/transforms.py index 8a35e6c3b908e..27eca19c28be6 100644 --- a/python/paddle/vision/transforms/transforms.py +++ b/python/paddle/vision/transforms/transforms.py @@ -848,13 +848,13 @@ class ColorJitter(BaseTransform): """Randomly change the brightness, contrast, saturation and hue of an image. Args: - brightness: How much to jitter brightness. + brightness (float): How much to jitter brightness. Chosen uniformly from [max(0, 1 - brightness), 1 + brightness]. Should be non negative numbers. - contrast: How much to jitter contrast. + contrast (float): How much to jitter contrast. Chosen uniformly from [max(0, 1 - contrast), 1 + contrast]. Should be non negative numbers. - saturation: How much to jitter saturation. + saturation (float): How much to jitter saturation. Chosen uniformly from [max(0, 1 - saturation), 1 + saturation]. Should be non negative numbers. - hue: How much to jitter hue. + hue (float): How much to jitter hue. Chosen uniformly from [-hue, hue]. Should have 0<= hue <= 0.5. keys (list[str]|tuple[str], optional): Same as ``BaseTransform``. Default: None. From a43e1fac7ac31797caae2730929e824691b0a85a Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Sat, 12 Jun 2021 21:21:04 +0800 Subject: [PATCH 104/156] Fix LayerNorm Problem Release2.1 (#33534) * Eliminate numerical differences of LayerNorm; fix LayerNorm Nan Bug while large data input * fix bug while large shape of data input --- paddle/fluid/operators/layer_norm_op.cu | 202 ++++++++++-------- .../tests/unittests/test_layer_norm_op_v2.py | 1 + 2 files changed, 115 insertions(+), 88 deletions(-) mode change 100644 => 100755 paddle/fluid/operators/layer_norm_op.cu diff --git a/paddle/fluid/operators/layer_norm_op.cu b/paddle/fluid/operators/layer_norm_op.cu old mode 100644 new mode 100755 index 3656de3525d32..f955011675cf5 --- a/paddle/fluid/operators/layer_norm_op.cu +++ b/paddle/fluid/operators/layer_norm_op.cu @@ -42,15 +42,46 @@ using CudnnDataType = platform::CudnnDataType; template using LayerNormParamType = typename CudnnDataType::BatchNormParamType; -inline static int GetDesiredBlockDim(int block_dim) { +inline static int GetDesiredBlockDim(int64_t block_dim) { #ifdef __HIPCC__ const int kMaxBlockDim = 256; + const int lwarpSize = 64; #else const int kMaxBlockDim = 512; + const int lwarpSize = 32; #endif - return block_dim >= kMaxBlockDim - ? kMaxBlockDim - : (1 << (static_cast(std::log2f(block_dim)))); + return block_dim >= kMaxBlockDim ? kMaxBlockDim : lwarpSize; +} + +template +static __forceinline__ __device__ U WarpReduceSum(U val) { + unsigned mask = 0u; + CREATE_SHFL_MASK(mask, true); + for (int offset = warpSize / 2; offset > 0; offset /= 2) { + val += paddle::platform::CudaShuffleDownSync(mask, val, offset); + } + return val; +} + +template +__forceinline__ __device__ U BlockReduceSum(U val) { + static __shared__ U shared[32]; + int lane = threadIdx.x % warpSize; + int wid = threadIdx.x / warpSize; + + val = WarpReduceSum(val); // Each warp performs partial reduction + + if (lane == 0) shared[wid] = val; // Write reduced value to shared memory + + __syncthreads(); // Wait for all partial reductions + + // read from shared memory only if that warp existed + val = + (threadIdx.x < blockDim.x / warpSize) ? shared[lane] : static_cast(0); + + if (wid == 0) val = WarpReduceSum(val); // Final reduce within first warp + + return val; } #define FIXED_BLOCK_DIM_CASE_BASE(log2_block_dim, ...) \ @@ -70,15 +101,17 @@ inline static int GetDesiredBlockDim(int block_dim) { FIXED_BLOCK_DIM_CASE_BASE(2, ##__VA_ARGS__); \ FIXED_BLOCK_DIM_CASE_BASE(1, ##__VA_ARGS__) -#define FIXED_BLOCK_DIM_FIXED_BLOCK_NUM_CASE_BASE( \ - log2_block_dim, feature_size, kMaxBlockNum, ...) \ - case (1 << (log2_block_dim)): { \ - for (int i = 0; i < std::ceil(feature_size / (1.0 * kMaxBlockNum)); i++) { \ - int col_offset = i * kMaxBlockNum; \ - int block_num = std::min(feature_size - col_offset, kMaxBlockNum); \ - constexpr auto kBlockDim = (1 << (log2_block_dim)); \ - __VA_ARGS__; \ - } \ +#define FIXED_BLOCK_DIM_FIXED_BLOCK_NUM_CASE_BASE( \ + log2_block_dim, feature_size, kMaxBlockNum, ...) \ + case (1 << (log2_block_dim)): { \ + for (int64_t i = 0; i < std::ceil(feature_size / (1.0 * kMaxBlockNum)); \ + i++) { \ + int64_t col_offset = i * static_cast(kMaxBlockNum); \ + int block_num = static_cast(std::min( \ + feature_size - col_offset, static_cast(kMaxBlockNum))); \ + constexpr auto kBlockDim = (1 << (log2_block_dim)); \ + __VA_ARGS__; \ + } \ } break #define FIXED_BLOCK_DIM_FIXED_BLOCK_NUM_CASE(feature_size, kMaxBlockNum, ...) \ @@ -147,31 +180,32 @@ __inline__ __device__ half rsqrt_(const half val) { template __global__ void LayerNormForward(const T *x, const U *scale, const U *bias, T *y, U *mean, U *var, float epsilon, - int feature_size) { - using BlockReduce = cub::BlockReduce, BlockDim>; - __shared__ typename BlockReduce::TempStorage temp_storage; + int64_t feature_size) { __shared__ U mean_share; __shared__ U var_share; - int beg_idx = blockIdx.x * feature_size + threadIdx.x; - int end_idx = (blockIdx.x + 1) * feature_size; + int64_t beg_idx = blockIdx.x * feature_size + threadIdx.x; + int64_t end_idx = (blockIdx.x + 1) * feature_size; // Step 1: Reduce to calculate mean and var U mean_val = 0; U var_val = 0; - for (int i = beg_idx; i < end_idx; i += BlockDim) { + for (int64_t i = beg_idx; i < end_idx; i += BlockDim) { U tmp = static_cast(x[i]); mean_val += tmp; var_val += (tmp * tmp); } - auto pair = BlockReduce(temp_storage) - .Reduce(PairForLayerNorm(mean_val, var_val), - PairForLayerNormAddFunctor()); + + mean_val = BlockReduceSum(mean_val); + var_val = BlockReduceSum(var_val); + if (threadIdx.x == 0) { - auto tmp = pair.first_ / feature_size; + auto scale = static_cast(1.) / static_cast(feature_size); + auto tmp = mean_val * scale; mean[blockIdx.x] = mean_share = static_cast(tmp); - var[blockIdx.x] = var_share = - static_cast(pair.second_ / feature_size - tmp * tmp); + var_share = static_cast(var_val * scale - mean_share * mean_share); + var_share = var_share > U(0) ? var_share : U(0); + var[blockIdx.x] = var_share; } __syncthreads(); @@ -181,13 +215,13 @@ __global__ void LayerNormForward(const T *x, const U *scale, const U *bias, // Step 2: Calculate y if (scale != nullptr) { if (bias != nullptr) { - for (int i = beg_idx, j = threadIdx.x; i < end_idx; + for (int64_t i = beg_idx, j = threadIdx.x; i < end_idx; i += BlockDim, j += BlockDim) { y[i] = static_cast( scale[j] * (static_cast(x[i]) - mean_val) * invvar + bias[j]); } } else { - for (int i = beg_idx, j = threadIdx.x; i < end_idx; + for (int64_t i = beg_idx, j = threadIdx.x; i < end_idx; i += BlockDim, j += BlockDim) { y[i] = static_cast(scale[j] * (static_cast(x[i]) - mean_val) * invvar); @@ -195,13 +229,13 @@ __global__ void LayerNormForward(const T *x, const U *scale, const U *bias, } } else { // scale == nullptr if (bias != nullptr) { - for (int i = beg_idx, j = threadIdx.x; i < end_idx; + for (int64_t i = beg_idx, j = threadIdx.x; i < end_idx; i += BlockDim, j += BlockDim) { y[i] = static_cast((static_cast(x[i]) - mean_val) * invvar + bias[j]); } } else { - for (int i = beg_idx, j = threadIdx.x; i < end_idx; + for (int64_t i = beg_idx, j = threadIdx.x; i < end_idx; i += BlockDim, j += BlockDim) { y[i] = static_cast((static_cast(x[i]) - mean_val) * invvar); } @@ -211,18 +245,18 @@ __global__ void LayerNormForward(const T *x, const U *scale, const U *bias, template __inline__ __device__ void cuLoadAddStridedInputs( - const int i1_block, const int thr_load_row_off, const int thr_load_col_off, - const int i2_off, const int row_stride, U *warp_buf1, U *warp_buf2, - const T *input, const T *dout, const int i1_end, const int n2, - const U *__restrict__ mean, const U *__restrict__ var, - const float epsilon) { - const int i1 = i1_block + thr_load_row_off; + const int64_t i1_block, const int thr_load_row_off, + const int thr_load_col_off, const int i2_off, const int row_stride, + U *warp_buf1, U *warp_buf2, const T *input, const T *dout, + const int64_t i1_end, const int64_t n2, const U *__restrict__ mean, + const U *__restrict__ var, const float epsilon) { + const int64_t i1 = i1_block + thr_load_row_off; if (i1 >= i1_end) return; U curr_mean = mean[i1]; U curr_invvar = rsqrt_(var[i1] + epsilon); for (int k = 0; k < VPT; ++k) { const int i2 = i2_off + k; - const int load_idx = i1 * n2 + i2; + const int64_t load_idx = i1 * n2 + i2; const int write_idx = thr_load_row_off * row_stride + thr_load_col_off + k; if (i2 < n2) { U curr_input = static_cast(input[load_idx]); @@ -236,8 +270,8 @@ __inline__ __device__ void cuLoadAddStridedInputs( template __global__ void LayerNormBackwardPartGradGammaBeta( - const T *__restrict__ dout, const T *__restrict__ input, const int n1, - const int n2, const U *__restrict__ mean, const U *__restrict__ var, + const T *__restrict__ dout, const T *__restrict__ input, const int64_t n1, + const int64_t n2, const U *__restrict__ mean, const U *__restrict__ var, float epsilon, U *part_grad_gamma, U *part_grad_beta) { // VPTX -> value per thread.x, BDIMX -> blockDim.x, BDIMY -> blockDim.y, BDIMX // -> blockDim.x @@ -263,7 +297,7 @@ __global__ void LayerNormBackwardPartGradGammaBeta( } __syncthreads(); - for (int i1_block = blockIdx.y * BDIMY * VPTX; i1_block < n1; + for (int64_t i1_block = blockIdx.y * BDIMY * VPTX; i1_block < n1; i1_block += VPTX * BDIMY * gridDim.y) { cuLoadAddStridedInputs( i1_block, thr_load_row_off, thr_load_col_off, i2_off, row_stride, @@ -296,7 +330,7 @@ __global__ void LayerNormBackwardPartGradGammaBeta( } __syncthreads(); } - int i2 = blockIdx.x * blockDim.x + threadIdx.x; + int64_t i2 = blockIdx.x * blockDim.x + threadIdx.x; if (threadIdx.y == 0 && i2 < n2) { int row1 = threadIdx.y; int row2 = threadIdx.y + 1; @@ -314,7 +348,7 @@ __global__ void LayerNormBackwardSumGradGammaBeta( const int n1, const int n2, U *grad_gamma, U *grad_beta) { // sum partial gradients for gamma and beta __shared__ U buf[BDIMX * BDIMY]; - int i2 = blockIdx.x * BDIMX + threadIdx.x; + int64_t i2 = blockIdx.x * BDIMX + threadIdx.x; if (i2 < n2) { // each warp does sequential reductions until reduced part_size is num_warps int num_warp_reductions = part_size / BDIMY; @@ -485,22 +519,17 @@ __global__ void LayerNormBackwardComputeGradInput( // Make sure that d_scale != nullptr && d_bias != nullptr // Since d_scale != nullptr, scale would not be nullptr template -__global__ void LayerNormBackwardGradientAll(const T *x, const T *d_y, - U *d_scale, U *d_bias, T *d_x, - const U *mean, const U *var, - const U *scale, float epsilon, - int batch_size, int feature_size, - int col_offset) { - using BlockReduce = cub::BlockReduce, BlockDim>; - __shared__ typename BlockReduce::TempStorage temp_storage; - - int beg_idx = threadIdx.x * feature_size + (blockIdx.x + col_offset); - int end_idx = batch_size * feature_size + (blockIdx.x + col_offset); - int stride = BlockDim * feature_size; +__global__ void LayerNormBackwardGradientAll( + const T *x, const T *d_y, U *d_scale, U *d_bias, T *d_x, const U *mean, + const U *var, const U *scale, float epsilon, int64_t batch_size, + int64_t feature_size, int64_t col_offset) { + int64_t beg_idx = threadIdx.x * feature_size + (blockIdx.x + col_offset); + int64_t end_idx = batch_size * feature_size + (blockIdx.x + col_offset); + int64_t stride = BlockDim * feature_size; U d_scale_partial = static_cast(0), d_bias_partial = static_cast(0); - for (int i = beg_idx; i < end_idx; i += stride) { + for (int64_t i = beg_idx; i < end_idx; i += stride) { int row_idx = i / feature_size; auto var_val = real_sqrt(static_cast(var[row_idx]) + epsilon); d_scale_partial += static_cast(d_y[i]) * @@ -512,13 +541,12 @@ __global__ void LayerNormBackwardGradientAll(const T *x, const T *d_y, } } - auto pair = BlockReduce(temp_storage) - .Reduce(PairForLayerNorm(d_scale_partial, d_bias_partial), - PairForLayerNormAddFunctor()); + d_scale_partial = BlockReduceSum(d_scale_partial); + d_bias_partial = BlockReduceSum(d_bias_partial); if (threadIdx.x == 0) { - d_scale[blockIdx.x + col_offset] = pair.first_; - d_bias[blockIdx.x + col_offset] = pair.second_; + d_scale[blockIdx.x + col_offset] = d_scale_partial; + d_bias[blockIdx.x + col_offset] = d_bias_partial; } } @@ -528,16 +556,16 @@ __global__ void LayerNormBackwardGradientAll(const T *x, const T *d_y, template __global__ void LayerNormBackwardGradientScaleOrBias( const T *x, const T *d_y, U *d_scale, U *d_bias, T *d_x, const U *mean, - const U *var, const U *scale, float epsilon, int batch_size, - int feature_size, int col_offset) { + const U *var, const U *scale, float epsilon, int64_t batch_size, + int64_t feature_size, int col_offset) { using BlockReduce = cub::BlockReduce; __shared__ typename BlockReduce::TempStorage temp_storage; - int beg_idx = threadIdx.x * feature_size + blockIdx.x + col_offset; - int end_idx = batch_size * feature_size + blockIdx.x + col_offset; + int64_t beg_idx = threadIdx.x * feature_size + blockIdx.x + col_offset; + int64_t end_idx = batch_size * feature_size + blockIdx.x + col_offset; int stride = BlockDim * feature_size; U d_scale_or_d_bias_partial = static_cast(0); - for (int i = beg_idx; i < end_idx; i += stride) { + for (int64_t i = beg_idx; i < end_idx; i += stride) { int row_idx = i / feature_size; auto var_val = static_cast(real_sqrt(static_cast(var[row_idx]) + epsilon)); @@ -572,22 +600,20 @@ __global__ void LayerNormBackwardGradientScaleOrBias( } template -__global__ void LayerNormBackwardPostProcessToCalculateDX(const T *x, T *d_x, - const U *mean, - const U *var, - float epsilon, - int feature_size) { +__global__ void LayerNormBackwardPostProcessToCalculateDX( + const T *x, T *d_x, const U *mean, const U *var, float epsilon, + int64_t feature_size) { using BlockReduce = cub::BlockReduce, BlockDim>; __shared__ typename BlockReduce::TempStorage temp_storage; __shared__ U d_x_reduce_tmp[2]; - int beg_idx = blockIdx.x * feature_size + threadIdx.x; - int end_idx = (blockIdx.x + 1) * feature_size; + int64_t beg_idx = blockIdx.x * feature_size + threadIdx.x; + int64_t end_idx = (blockIdx.x + 1) * feature_size; U block_mean = mean[blockIdx.x]; U block_var = var[blockIdx.x]; U d_x_mean_partial = static_cast(0), d_x_var_partial = static_cast(0); - for (int i = beg_idx; i < end_idx; i += BlockDim) { + for (int64_t i = beg_idx; i < end_idx; i += BlockDim) { d_x_mean_partial += static_cast(d_x[i]); d_x_var_partial += static_cast(d_x[i]) * (static_cast(x[i]) - block_mean); @@ -608,7 +634,7 @@ __global__ void LayerNormBackwardPostProcessToCalculateDX(const T *x, T *d_x, d_x_mean_partial = d_x_reduce_tmp[0]; d_x_var_partial = d_x_reduce_tmp[1]; - for (int i = beg_idx; i < end_idx; i += BlockDim) { + for (int64_t i = beg_idx; i < end_idx; i += BlockDim) { d_x[i] -= static_cast(d_x_mean_partial); d_x[i] -= static_cast((static_cast(x[i]) - block_mean) * d_x_var_partial); @@ -621,17 +647,17 @@ __global__ void LayerNormBackwardGradientOnlyDX(const T *x, const T *d_y, T *d_x, const U *mean, const U *var, const U *scale, float epsilon, - int feature_size) { + int64_t feature_size) { using BlockReduce = cub::BlockReduce, BlockDim>; __shared__ typename BlockReduce::TempStorage temp_storage; __shared__ U d_x_reduce_tmp[2]; - int beg_idx = blockIdx.x * feature_size + threadIdx.x; - int end_idx = (blockIdx.x + 1) * feature_size; + int64_t beg_idx = blockIdx.x * feature_size + threadIdx.x; + int64_t end_idx = (blockIdx.x + 1) * feature_size; U block_mean = mean[blockIdx.x], block_var = var[blockIdx.x]; U d_x_mean_partial = static_cast(0), d_x_var_partial = static_cast(0); - for (int i = beg_idx; i < end_idx; i += BlockDim) { + for (int64_t i = beg_idx; i < end_idx; i += BlockDim) { auto var_val = static_cast(real_sqrt(static_cast(block_var) + epsilon)); if (scale != nullptr) { @@ -661,7 +687,7 @@ __global__ void LayerNormBackwardGradientOnlyDX(const T *x, const T *d_y, d_x_mean_partial = d_x_reduce_tmp[0]; d_x_var_partial = d_x_reduce_tmp[1]; - for (int i = beg_idx; i < end_idx; i += BlockDim) { + for (int64_t i = beg_idx; i < end_idx; i += BlockDim) { d_x[i] -= static_cast(d_x_mean_partial); d_x[i] -= static_cast((static_cast(x[i]) - block_mean) * d_x_var_partial); @@ -671,8 +697,8 @@ __global__ void LayerNormBackwardGradientOnlyDX(const T *x, const T *d_y, template __global__ void LayerNormBackwardWhenBatchSizeIsOne( const T *x, const T *d_y, T *d_x, U *d_scale, U *d_bias, const U *mean, - const U *var, const U *scale, float epsilon, int feature_size) { - int idx = threadIdx.x + blockIdx.x * blockDim.x; + const U *var, const U *scale, float epsilon, int64_t feature_size) { + int64_t idx = threadIdx.x + blockIdx.x * blockDim.x; if (idx < feature_size) { auto var_val = static_cast(real_sqrt(static_cast(var[idx]) + epsilon)); @@ -697,8 +723,8 @@ __global__ void LayerNormBackwardWhenBatchSizeIsOne( template static void LayerNormBackward(const T *x, const T *d_y, const U *scale, const U *mean, const U *var, T *d_x, U *d_scale, - U *d_bias, float epsilon, int batch_size, - int feature_size, + U *d_bias, float epsilon, int64_t batch_size, + int64_t feature_size, const framework::ExecutionContext &ctx) { auto &dev_ctx = ctx.cuda_device_context(); auto stream = dev_ctx.stream(); @@ -858,8 +884,8 @@ void LayerNormDirectCUDAFunctor::operator()(gpuStream_t stream, int begin_norm_axis, float eps) { const auto x_dims = framework::make_ddim(input_shape); auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int batch_size = static_cast(matrix_dim[0]); - int feature_size = static_cast(matrix_dim[1]); + int64_t batch_size = static_cast(matrix_dim[0]); + int64_t feature_size = static_cast(matrix_dim[1]); switch (GetDesiredBlockDim(feature_size)) { FIXED_BLOCK_DIM_CASE( LayerNormForward<<>>( @@ -897,8 +923,8 @@ class LayerNormKernel auto *bias_data = (bias == nullptr ? nullptr : bias->data()); auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int batch_size = static_cast(matrix_dim[0]); - int feature_size = static_cast(matrix_dim[1]); + int64_t batch_size = static_cast(matrix_dim[0]); + int64_t feature_size = static_cast(matrix_dim[1]); auto stream = ctx.cuda_device_context().stream(); @@ -951,8 +977,8 @@ class LayerNormGradKernel const auto &x_dims = x->dims(); const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int batch_size = static_cast(matrix_dim[0]); - int feature_size = static_cast(matrix_dim[1]); + int64_t batch_size = static_cast(matrix_dim[0]); + int64_t feature_size = static_cast(matrix_dim[1]); LayerNormBackward(x_data, d_y_data, scale_data, mean_data, var_data, d_x_data, d_scale_data, d_bias_data, epsilon, diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py index 77cd6926b563d..987c3da4dd7be 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py @@ -51,6 +51,7 @@ def compute_v2(x): self.assertTrue(np.allclose(y1, y2)) def test_static(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("layer_norm"): places.append(fluid.CUDAPlace(0)) From f7034613f7be66c3b2a7fb5720d40e74cdca811a Mon Sep 17 00:00:00 2001 From: wenbin Date: Tue, 15 Jun 2021 08:49:59 +0800 Subject: [PATCH 105/156] refix if-else logic for inference: missing if (#33531) --- paddle/fluid/inference/api/analysis_predictor.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index baff7a6f57c52..42793595e19c8 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -303,7 +303,9 @@ static void DisablePrepareDataOpt( disable_opt || pre_disable_opt); } // disable prepare data if unfriendly op is found - disable_opt = IsPrepareDataOptTargetOp(op); + if (!disable_opt) { + disable_opt = IsPrepareDataOptTargetOp(op); + } } } From 0079e0b1af7463315bf019136d3776b6924cdc6a Mon Sep 17 00:00:00 2001 From: WeiXin Date: Tue, 15 Jun 2021 11:34:16 +0800 Subject: [PATCH 106/156] [Cherry-Pick] Fix the segfault when using to_tensor in PyLayer. (#33303) (#33518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复pylayer 返回to_tensor时触发段错误的bug。 原因： 如果在Python端修改了stop_gradient属性，c++ 端InnerSetOverridedStopGradient 无法修改stop_gradient属性，在c++端调用SetOverridedStopGradient修改stop_gradient属性。 to_tensor产生的tensor的grad var的DataType为默认值（-1），在backward的过程中grad var的DataType不能为默认值（-1），因此在调用ForwardDataType设置grad var的DataType。 原始PR：#33303 --- paddle/fluid/imperative/py_layer_fwd.h | 67 +++++---- paddle/fluid/operators/py_layer_op.cc | 42 ++++-- .../fluid/tests/unittests/test_pylayer_op.py | 128 ++++++++++++++++++ 3 files changed, 202 insertions(+), 35 deletions(-) diff --git a/paddle/fluid/imperative/py_layer_fwd.h b/paddle/fluid/imperative/py_layer_fwd.h index de5f9d75e9173..1baf73ab3b95d 100644 --- a/paddle/fluid/imperative/py_layer_fwd.h +++ b/paddle/fluid/imperative/py_layer_fwd.h @@ -17,6 +17,7 @@ #include #include #include "paddle/fluid/imperative/layer.h" +#include "paddle/fluid/imperative/prepared_operator.h" #include "paddle/fluid/imperative/tracer.h" #include "paddle/fluid/framework/op_registry.h" @@ -32,7 +33,17 @@ bool RequiredGrad(const NameVarBaseMap& ins, const NameVarBaseMap& outs) { for (const auto& name_pair : ins) { for (const auto& var_base : name_pair.second) { if (!var_base->OverridedStopGradient()) { - PassStopGradient(outs, var_base->OverridedStopGradient()); + for (const auto& pair : outs) { + for (const auto& var : pair.second) { + if (var) { + var->SetOverridedStopGradient(false); + SetForwardDataTypeOfGradVar(var); + VLOG(3) << "Set output: " << var->Name() + << "'s OverridedStopGradient as " + << var->OverridedStopGradient(); + } + } + } return true; } } @@ -78,28 +89,36 @@ py::object PyLayerApply(const platform::Place& place, const py::handle& cls, // process args,`input_vars` only collect `imperative::VarBase` if (!args.empty()) { for (auto ptr = args.begin(); ptr != args.end(); ptr++) { - try { - if (Py_None != ptr->ptr()) { + // Only collect Tensor type in 'args' and pass them to backward. Ignore + // other types of input temporarily. + if (py::isinstance(*ptr)) { + try { auto a = ptr->cast>(); input_vars.push_back(a); + } catch (py::cast_error& err) { + PADDLE_THROW(platform::errors::InvalidArgument( + "The `PyLayer.forward` function contains invalid argument, the " + "`%s` type argument can not be cast into `Tensor`.", + ptr->ptr()->ob_type->tp_name)); } - } catch (py::cast_error& err) { - // Only collect Tensor type in 'args' and pass them to backward. Ignore - // other types of input temporarily. } } } // process kwargs, only collect `imperative::VarBase` if (!kwargs.empty()) { for (auto ptr = kwargs.begin(); ptr != kwargs.end(); ptr++) { - try { - if (Py_None != ptr->second.ptr()) { + // Only collect Tensor type in 'kwargs' and pass them to backward. + // Ignore other types of input temporarily. + if (py::isinstance(*ptr->second)) { + try { auto a = ptr->second.cast>(); input_vars.push_back(a); + } catch (py::cast_error&) { + PADDLE_THROW(platform::errors::InvalidArgument( + "The `PyLayer.forward` function contains invalid argument, the " + "`%s` type argument can not be cast into `Tensor`.", + ptr->second.ptr()->ob_type->tp_name)); } - } catch (py::cast_error&) { - // Only collect Tensor type in 'kwargs' and pass them to backward. - // Ignore other types of input temporarily. } } } @@ -110,33 +129,35 @@ py::object PyLayerApply(const platform::Place& place, const py::handle& cls, PyList_Check(result_forward.ptr())) { auto tuple_result = result_forward.cast(); for (size_t i = 0; i < tuple_result.size(); i++) { - if (Py_None != tuple_result[i].ptr()) { + // Only collect Tensor type of output and pass them to backward. + // Ignore other types of input temporarily. + if (py::isinstance(tuple_result[i])) { try { auto temp_out = tuple_result[i].cast>(); output_vars.push_back(temp_out); } catch (py::cast_error&) { - // Only collect Tensor type in 'kwargs' and pass them to backward. - // Ignore other types of input temporarily. + PADDLE_THROW(platform::errors::InvalidArgument( + "The `PyLayer.forward` function returns invalid argument, the " + "`%s` type argument can not be cast into `Tensor`.", + tuple_result[i].ptr()->ob_type->tp_name)); } - } else { - // Only collect Tensor type in 'kwargs' and pass them to backward. - // Ignore other types of input temporarily. } } } else { - if (Py_None != result_forward.ptr()) { + // Only collect Tensor type of output and pass them to backward. + // Ignore other types of input temporarily. + if (py::isinstance(result_forward)) { try { auto temp_out = result_forward.cast>(); output_vars.push_back(temp_out); } catch (py::cast_error&) { - // Only collect Tensor type in 'kwargs' and pass them to backward. - // Ignore other types of input temporarily. + PADDLE_THROW(platform::errors::InvalidArgument( + "The `PyLayer.forward` function returns invalid argument, the `%s` " + "type argument can not be cast into `Tensor`.", + result_forward.ptr()->ob_type->tp_name)); } - } else { - // Only collect Tensor type in 'kwargs' and pass them to backward. - // Ignore other types of input temporarily. } } if (output_vars.size() == 0) { diff --git a/paddle/fluid/operators/py_layer_op.cc b/paddle/fluid/operators/py_layer_op.cc index f91496eeab142..4261b72f1465a 100644 --- a/paddle/fluid/operators/py_layer_op.cc +++ b/paddle/fluid/operators/py_layer_op.cc @@ -62,13 +62,22 @@ void RunPyObject(py::object *py_object, for (size_t i = 0; i < result_tuple.size(); i++) { if ((*outs)[i] != nullptr) { if (Py_None != result_tuple[i].ptr()) { - try { - auto result_var = - result_tuple[i].cast>(); - *(*outs)[i] = result_var->Var(); - } catch (py::cast_error &) { + if (py::isinstance(result_tuple[i])) { + try { + auto result_var = + result_tuple[i].cast>(); + *(*outs)[i] = result_var->Var(); + } catch (py::cast_error &) { + PADDLE_THROW(platform::errors::InvalidArgument( + "The `PyLayer.backward` function returns invalid argument, " + "the `%s` type argument can not be cast into `Tensor`.", + result_tuple[i].ptr()->ob_type->tp_name)); + } + } else { PADDLE_THROW(platform::errors::InvalidArgument( - "The output of `PyLayer.backward` should be `Tensor`.")); + "The output of `PyLayer.backward` should be `Tensor`, but " + "received `%s`.", + result_tuple[i].ptr()->ob_type->tp_name)); } } else { PADDLE_THROW(platform::errors::InvalidArgument( @@ -94,13 +103,22 @@ void RunPyObject(py::object *py_object, } if ((*outs)[0] != nullptr) { if (Py_None != py_result.ptr()) { - try { - auto result_var = - py_result.cast>(); - *((*outs)[0]) = result_var->Var(); - } catch (py::cast_error &) { + if (py::isinstance(py_result)) { + try { + auto result_var = + py_result.cast>(); + *((*outs)[0]) = result_var->Var(); + } catch (py::cast_error &) { + PADDLE_THROW(platform::errors::InvalidArgument( + "The `PyLayer.backward` function returns invalid argument, the " + "`%s` type argument can not be cast into `Tensor`.", + py_result.ptr()->ob_type->tp_name)); + } + } else { PADDLE_THROW(platform::errors::InvalidArgument( - "The output of `PyLayer.backward` should be `Tensor`.")); + "The output of `PyLayer.backward` should be `Tensor`, but " + "received `%s`", + py_result.ptr()->ob_type->tp_name)); } } else { PADDLE_THROW(platform::errors::InvalidArgument( diff --git a/python/paddle/fluid/tests/unittests/test_pylayer_op.py b/python/paddle/fluid/tests/unittests/test_pylayer_op.py index e058115d69199..a852b4c90421a 100644 --- a/python/paddle/fluid/tests/unittests/test_pylayer_op.py +++ b/python/paddle/fluid/tests/unittests/test_pylayer_op.py @@ -21,6 +21,11 @@ from paddle.autograd import PyLayer +class FakeTensor(paddle.fluid.core.VarBase): + def __init__(self): + pass + + class TestPyLayer(unittest.TestCase): def test_simple_pylayer_multiple_output(self): class tanh(PyLayer): @@ -426,6 +431,129 @@ def backward(ctx, dy): z = paddle.tanh(data) z = cus_tanh.apply(data) + def test_return_to_tensor(self): + class Tanh(PyLayer): + @staticmethod + def forward(ctx, x1): + y1 = paddle.tanh(x1) + ctx.save_for_backward(y1) + tensor_1 = paddle.to_tensor([1, 2], dtype='float32') + return y1, 5, None, "helloworld", tensor_1 + + @staticmethod + def backward(ctx, dy1, dy2): + y1, = ctx.saved_tensor() + re1 = dy1 * (1 - paddle.square(y1)) + return dy1 + + input1 = paddle.randn([2, 3]).astype("float32") + input2 = input1.detach().clone() + input1.stop_gradient = False + input2.stop_gradient = False + z, number, none_item, string_item, tensor1 = Tanh.apply(x1=input1) + z.mean().backward() + + +class TestPyLayerReturnType(unittest.TestCase): + def test_forward_args_fake_tensor(self): + class Tanh(PyLayer): + @staticmethod + def forward(ctx, x1): + y1 = FakeTensor() + return y1, x1 + + @staticmethod + def backward(ctx, dy1, dy2): + return dy1 + + input1 = FakeTensor() + + with self.assertRaises(ValueError): + y1, y2 = Tanh.apply(input1) + + def test_forward_kwargs_fake_tensor(self): + class Tanh(PyLayer): + @staticmethod + def forward(ctx, x1): + + return x1 + + @staticmethod + def backward(ctx, dy1, dy2): + return dy1 + + input1 = FakeTensor() + + with self.assertRaises(ValueError): + y = Tanh.apply(x1=input1) + + def test_forward_return_fake_tensor(self): + class Tanh(PyLayer): + @staticmethod + def forward(ctx, x1): + + return FakeTensor() + + @staticmethod + def backward(ctx, dy1, dy2): + return dy1 + + input1 = paddle.randn([3, 2]) + + with self.assertRaises(ValueError): + y = Tanh.apply(x1=input1) + + def test_forward_return_fake_tensor_tuple(self): + class Tanh(PyLayer): + @staticmethod + def forward(ctx, x1): + + return FakeTensor(), FakeTensor() + + @staticmethod + def backward(ctx, dy1, dy2): + return dy1 + + input1 = paddle.randn([3, 2]) + + with self.assertRaises(ValueError): + y = Tanh.apply(x1=input1) + + def test_backward_return_fake_tensor_tuple(self): + class Tanh(PyLayer): + @staticmethod + def forward(ctx, x1, x2): + return x1 + 1, x1 + 2 + + @staticmethod + def backward(ctx, dy1, dy2): + + return FakeTensor(), 2 + + input1 = paddle.randn([3, 2]) + input1.stop_gradient = False + y, _ = Tanh.apply(input1, 1 + input1) + + with self.assertRaises(ValueError): + y.mean().backward() + + def test_backward_return_fake_tensor(self): + class Tanh(PyLayer): + @staticmethod + def forward(ctx, x1): + return x1 + 1, x1 + 2 + + @staticmethod + def backward(ctx, dy1, dy2): + return FakeTensor() + + input1 = paddle.randn([3, 2]) + input1.stop_gradient = False + y, _ = Tanh.apply(input1) + + with self.assertRaises(ValueError): + y.mean().backward() + if __name__ == '__main__': unittest.main() From bbedca46f07d74c5d52b294f0fcc0470d90ec683 Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Tue, 15 Jun 2021 14:08:53 +0800 Subject: [PATCH 107/156] [cherry pick] add warning for dataloader incompatable upgrade (#33514) * add warning log for DataLoader output format imcompatible upgrade. test=develop * add unittest. test=develop * fix ci converage. test=develop * fix ci coverage. test=develop --- python/paddle/fluid/dataloader/fetcher.py | 43 +++++++++++++++ .../test_multiprocess_dataloader_dataset.py | 53 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/python/paddle/fluid/dataloader/fetcher.py b/python/paddle/fluid/dataloader/fetcher.py index 41e12fbc68ec1..05382b04dc457 100644 --- a/python/paddle/fluid/dataloader/fetcher.py +++ b/python/paddle/fluid/dataloader/fetcher.py @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging +from ..log_helper import get_logger + +from collections.abc import Sequence + class _DatasetFetcher(object): def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): @@ -19,11 +24,39 @@ def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): self.auto_collate_batch = auto_collate_batch self.collate_fn = collate_fn self.drop_last = drop_last + self._is_warning_logged = False def fetch(self, batch_indices): raise NotImplementedError("'fetch' not implement for class {}".format( self.__class__.__name__)) + def _log_warning(self): + warn_str = "Detect dataset only contains single fileds, return format " \ + "changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add " \ + "a list surround output data(e.g. return [data]), and in " \ + "Paddle >= 2.1, DataLoader return the single filed directly " \ + "(e.g. return data). For example, in following code: \n\n" + warn_str += \ + "import numpy as np\n" \ + "from paddle.io import DataLoader, Dataset\n\n" \ + "class RandomDataset(Dataset):\n" \ + " def __getitem__(self, idx):\n" \ + " data = np.random.random((2, 3)).astype('float32')\n\n" \ + " return data\n\n" \ + " def __len__(self):\n" \ + " return 10\n\n" \ + "dataset = RandomDataset()\n" \ + "loader = DataLoader(dataset, batch_size=1)\n" \ + "data = next(loader())\n\n" + + warn_str += "In Paddle <= 2.0, data is in format '[Tensor(shape=(1, 2, 3), " \ + "dtype=float32)]', and in Paddle >= 2.1, data is in format" \ + " 'Tensor(shape=(1, 2, 3), dtype=float32)'\n" + + logger = get_logger( + "DataLoader", logging.INFO, fmt='%(levelname)s: %(message)s') + logger.warning(warn_str) + class _IterableDatasetFetcher(_DatasetFetcher): def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): @@ -40,9 +73,14 @@ def fetch(self, batch_indices): data.append(next(self.dataset_iter)) except StopIteration: break + if len(data) == 0 or (self.drop_last and len(data) < len(batch_indices)): raise StopIteration + if not isinstance(data[0], + Sequence) and not self._is_warning_logged: + self._log_warning() + self._is_warning_logged = True else: data = next(self.dataset_iter) @@ -59,6 +97,11 @@ def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): def fetch(self, batch_indices): if self.auto_collate_batch: data = [self.dataset[idx] for idx in batch_indices] + + if not isinstance(data[0], + Sequence) and not self._is_warning_logged: + self._log_warning() + self._is_warning_logged = True else: data = self.dataset[batch_indices] diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py index 4c69d003d80f8..30e70a77c369c 100755 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py @@ -330,6 +330,59 @@ def test_main(self): self.run_main(num_workers) +class SingleFieldDataset(Dataset): + def __init__(self, sample_num): + self.sample_num = sample_num + + def __len__(self): + return self.sample_num + + def __getitem__(self, idx): + return np.random.random((2, 3)).astype('float32') + + +class TestSingleFieldDataset(unittest.TestCase): + def init_dataset(self): + self.sample_num = 16 + self.dataset = SingleFieldDataset(self.sample_num) + + def run_main(self, num_workers): + paddle.static.default_startup_program().random_seed = 1 + paddle.static.default_main_program().random_seed = 1 + place = paddle.CPUPlace() + with fluid.dygraph.guard(place): + self.init_dataset() + dataloader = DataLoader( + self.dataset, + places=place, + num_workers=num_workers, + batch_size=2, + drop_last=True) + + for i, data in enumerate(dataloader()): + assert isinstance(data, paddle.Tensor) + assert data.shape == [2, 2, 3] + + def test_main(self): + for num_workers in [0, 2]: + self.run_main(num_workers) + + +class SingleFieldIterableDataset(IterableDataset): + def __init__(self, sample_num): + self.sample_num = sample_num + + def __iter__(self): + for _ in range(self.sample_num): + yield np.random.random((2, 3)).astype('float32') + + +class TestSingleFieldIterableDataset(TestSingleFieldDataset): + def init_dataset(self): + self.sample_num = 16 + self.dataset = SingleFieldIterableDataset(self.sample_num) + + class TestDataLoaderGenerateStates(unittest.TestCase): def setUp(self): self.inputs = [(0, 1), (0, 2), (1, 3)] From 2b44ae5de9b4e53b2fd7ba992d353025dfc40b5c Mon Sep 17 00:00:00 2001 From: liym27 <33742067+liym27@users.noreply.github.com> Date: Tue, 15 Jun 2021 15:39:03 +0800 Subject: [PATCH 108/156] [cherry-pick] Polish code for setitem/getitem and support index for list/Tensor/None/Ellipsis/bool (#33528) * [cherry-pick 2.1] Polish code for _getitem_impl_ (#32868) * [cherry-pick] Polish code for setitem and getitem (#32911) * [slice getitem] Support getitem idx is Tensor or List (#33000) * [getitem] Support index is None for getitem in static mode (#33001) * [Static getitem] Support static Variable getitem for Ellipsis index (#32876) * [static getitem]Support index is list bool for getitem in static mode (#33298) --- python/paddle/fluid/framework.py | 355 +--------------- .../fluid/tests/unittests/test_variable.py | 181 +++++++- python/paddle/fluid/variable_index.py | 390 ++++++++++++++++++ 3 files changed, 569 insertions(+), 357 deletions(-) create mode 100644 python/paddle/fluid/variable_index.py diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index bc8a06cb1ed89..03e7833aca1d8 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -39,6 +39,7 @@ import paddle.version as fluid_version import warnings import functools +from .variable_index import _getitem_impl_, _setitem_impl_ __all__ = [ 'Program', @@ -794,205 +795,6 @@ def __instancecheck__(cls, instance): return issubclass(t, Parameter) -def _getitem_impl_(var, item): - """ - Slice the variable. - - Args: - item(int/slice/tuple) : the index. - - Returns: - Sliced variable - """ - - if not isinstance(item, tuple): - item = [item] - - decrease_axis = [] - slice_axis = [] - slice_start = [] - slice_end = [] - slice_step = [] - use_strided_slice = False - reverse_axis = [] - target_block = default_main_program().current_block() - - def fill_constant(shape, value, force_cpu=False, out=None): - var.block.append_op( - type='fill_constant', - inputs={}, - outputs={'Out': [out]}, - attrs={ - 'shape': shape, - 'dtype': out.dtype, - 'value': float(value), - 'force_cpu': force_cpu - }) - out.stop_gradient = True - return out - - for dim, slice_item in enumerate(item): - if isinstance(slice_item, slice): - start = slice_item.start - end = slice_item.stop - step = slice_item.step - - if start is None and end is None and step is None: - continue - - if step is None: - step = 1 - - if start is None and end is None: - assert (step == -1) - reverse_axis.append(dim) - continue - - if start is None: - start = 0 - - if end is None: - end = 10000000 - - if step != 1: - use_strided_slice = True - - slice_axis.append(dim) - slice_start.append(start) - slice_end.append(end) - slice_step.append(step) - else: - decrease_axis.append(dim) - slice_axis.append(dim) - slice_start.append(slice_item) - slice_step.append(1) - if isinstance(slice_item, Variable): - temp_1 = var.block.create_var(dtype=slice_item.dtype) - fill_constant([1], 1, force_cpu=True, out=temp_1) - temp_end = target_block.create_var(dtype=slice_item.dtype) - target_block.append_op( - type='elementwise_add', - inputs={'X': slice_item, - 'Y': temp_1}, - outputs={'Out': temp_end}, - attrs={'axis': -1}) - slice_end.append(temp_end) - else: - slice_end.append(slice_item + 1 - if slice_item != -1 else 10000000) - - def contain_var(one_list): - for ele in one_list: - if isinstance(ele, Variable): - return True - return False - - def get_new_list_tensor(old_list): - new_list_tensor = [] - for dim in old_list: - if isinstance(dim, Variable): - dim.stop_gradient = True - new_list_tensor.append(dim) - else: - assert (isinstance(dim, int)) - temp_out = var.block.create_var(dtype='int64') - fill_constant([1], dim, force_cpu=True, out=temp_out) - new_list_tensor.append(temp_out) - return new_list_tensor - - inputs = {'Input': [var]} - attrs = { - 'axes': slice_axis, - 'starts': [], - 'ends': [], - 'decrease_axis': decrease_axis - } - if (use_strided_slice == True): - attrs['strides'] = [] - infer_flags = list(1 for i in range(len(slice_axis))) - - # starts - if contain_var(slice_start): - inputs['StartsTensorList'] = get_new_list_tensor(slice_start) - for i, dim in enumerate(slice_start): - if isinstance(dim, Variable): - attrs['starts'].append(-1) - infer_flags[i] = -1 - else: - attrs['starts'].append(dim) - else: - attrs['starts'] = slice_start - - # ends - if contain_var(slice_end): - inputs['EndsTensorList'] = get_new_list_tensor(slice_end) - for i, dim in enumerate(slice_end): - if isinstance(dim, Variable): - attrs['ends'].append(-1) - infer_flags[i] = -1 - else: - attrs['ends'].append(dim) - else: - attrs['ends'] = slice_end - - # strides - if use_strided_slice == True: - if contain_var(slice_step): - inputs['StridesTensorList'] = get_new_list_tensor(slice_step) - for i, dim in enumerate(slice_step): - if isinstance(dim, Variable): - attrs['strides'].append(-1) - infer_flags[i] = -1 - else: - attrs['strides'].append(dim) - else: - attrs['strides'] = slice_step - # infer_flags - attrs['infer_flags'] = infer_flags - - out = var - if use_strided_slice == False and len(slice_axis) > 0: - # append slice_op here - slice_out_var = target_block.create_var( - name=unique_name.generate_with_ignorable_key(var.name + "_slice"), - dtype=var.dtype) - - target_block.append_op( - type="slice", - inputs=inputs, - outputs={'Out': [slice_out_var]}, - attrs=attrs) - - out = slice_out_var - elif use_strided_slice == True and len(slice_axis) > 0: - strided_slice_out_var = target_block.create_var( - name=unique_name.generate_with_ignorable_key(var.name + - "_strided_slice"), - dtype=var.dtype) - target_block.append_op( - type="strided_slice", - inputs=inputs, - outputs={'Out': [strided_slice_out_var]}, - attrs=attrs) - - out = strided_slice_out_var - - if len(reverse_axis) > 0: - reverse_out_var = target_block.create_var( - name=unique_name.generate_with_ignorable_key(var.name + - "_slice_reverse"), - dtype=var.dtype) - target_block.append_op( - type="reverse", - inputs={'X': out}, - outputs={'Out': [reverse_out_var]}, - attrs={'axis': reverse_axis}) - - out = reverse_out_var - - return out - - @six.add_metaclass(VariableMetaClass) class Variable(object): """ @@ -1848,160 +1650,7 @@ def __getitem__(self, item): return _getitem_impl_(self, item) def __setitem__(self, item, value): - inputs = {'Input': self} - - # 1. Parse item - if not isinstance(item, tuple): - item = [item] - - decrease_axes = [] - axes = [] - starts = [] - ends = [] - steps = [] - - max_integer = sys.maxsize - - def replace_ellipsis(item): - # Use slice(None) to replace Ellipsis. - # For var, var.shape = [3,4,5,6] - # - # var[..., 1:2] -> var[:, :, :, 1:2] - # var[0, ...] -> var[0] - # var[0, ..., 1:2] -> var[0, :, :, 1:2] - - item = list(item) - - # Remove Variable to skip bug when counting Ellipsis - item_remove_var = [ - ele for ele in item if not isinstance(ele, Variable) - ] - ell_count = item_remove_var.count(Ellipsis) - if ell_count == 0: - return item - elif ell_count > 1: - raise IndexError( - "An index can only have a single ellipsis ('...')") - - ell_idx = item.index(Ellipsis) - - if ell_idx == len(item) - 1: - return item[:-1] - else: - item[ell_idx:ell_idx + 1] = [slice(None)] * ( - len(self.shape) - len(item) + 1) - - return item - - item = replace_ellipsis(item) - - for dim, slice_item in enumerate(item): - if isinstance(slice_item, slice): - start = slice_item.start - end = slice_item.stop - step = slice_item.step - - if start is None and end is None and step is None: - continue - - step = 1 if step is None else step - - # TODO: support cases when step < 1 - if not isinstance(step, Variable) and step == 0: - raise ValueError( - "When assign a value to a paddle.Tensor, step can not be 0, " - "but received step is {}.".format(step)) - - if isinstance(step, Variable) and (start is None or - end is None): - raise ValueError( - "When assign a value to a paddle.Tensor, it's not supported that " - "the start or end is None when the type of step is paddle.Tensor." - ) - - if start is None: - start = 0 if step > 0 else max_integer - - if end is None: - end = max_integer if step > 0 else (0 - max_integer) - else: - decrease_axes.append(dim) - start = slice_item - end = slice_item + 1 if slice_item != -1 else max_integer - step = 1 - - axes.append(dim) - starts.append(start) - ends.append(end) - steps.append(step) - - attrs = { - 'axes': axes, - 'starts': starts, - 'ends': ends, - 'steps': steps, - 'decrease_axes': decrease_axes - } - - from .layers import utils - if utils._contain_var(starts): - inputs['StartsTensorList'] = utils._convert_to_tensor_list(starts) - del attrs['starts'] - if utils._contain_var(ends): - inputs['EndsTensorList'] = utils._convert_to_tensor_list(ends) - del attrs['ends'] - if utils._contain_var(steps): - inputs['StepsTensorList'] = utils._convert_to_tensor_list(steps) - del attrs['steps'] - - # 2. Parse value - dtype = self.dtype - attrs['dtype'] = dtype - - from .data_feeder import convert_dtype - # 2.1 value is an integer of float - if isinstance(value, (int, float)): - value = np.array([value]).astype(convert_dtype(dtype)) - - # 2.2 value is a np.ndarray - if isinstance(value, np.ndarray): - shape = list(value.shape) - if dtype == core.VarDesc.VarType.BOOL: - value_name = "bool_values" - values = [bool(v) for v in value.flat] - elif dtype == core.VarDesc.VarType.FP32: - value_name = "fp32_values" - values = [float(v) for v in value.flat] - elif dtype == core.VarDesc.VarType.FP64: - value_name = "fp64_values" - values = [float(v) for v in value.flat] - elif dtype == core.VarDesc.VarType.INT32: - value_name = "int32_values" - values = [int(v) for v in value.flat] - elif dtype == core.VarDesc.VarType.INT64: - value_name = "int64_values" - values = [int(v) for v in value.flat] - else: - raise TypeError( - "When assign a numpy.ndarray, integer or float to a paddle.Tensor, " - "the data type of the paddle.Tensor must be bool, float32, int32 or int64, but " - "received %s." % convert_dtype(dtype)) - attrs[value_name] = values - attrs["shape"] = shape - - elif isinstance(value, Variable): - inputs["ValueTensor"] = value - else: - raise TypeError( - "Only support to assign an integer, float, numpy.ndarray or " - "paddle.Tensor to a paddle.Tensor, but received {}".format( - type(value))) - - cur_block = default_main_program().current_block() - cur_block.append_op( - type="set_value", inputs=inputs, outputs={'Out': self}, attrs=attrs) - - return self + return _setitem_impl_(self, item, value) def get_value(self, scope=None): """ diff --git a/python/paddle/fluid/tests/unittests/test_variable.py b/python/paddle/fluid/tests/unittests/test_variable.py index 690ac46e563ef..c1956545f55ad 100644 --- a/python/paddle/fluid/tests/unittests/test_variable.py +++ b/python/paddle/fluid/tests/unittests/test_variable.py @@ -15,12 +15,16 @@ from __future__ import print_function import unittest +import paddle from paddle.fluid.framework import default_main_program, Program, convert_np_dtype_to_dtype_, in_dygraph_mode +import paddle import paddle.fluid as fluid import paddle.fluid.layers as layers import paddle.fluid.core as core import numpy as np +paddle.enable_static() + class TestVariable(unittest.TestCase): def test_np_dtype_convert(self): @@ -161,12 +165,125 @@ def _test_slice(self, place): self.assertTrue( np.array_equal(local_out[15], tensor_array[::-1, ::-1, ::-1])) - def test_slice(self): - place = fluid.CPUPlace() - self._test_slice(place) + def _test_slice_index_tensor(self, place): + data = np.random.rand(2, 3).astype("float32") + prog = paddle.static.Program() + with paddle.static.program_guard(prog): + x = paddle.assign(data) + idx0 = [1, 0] + idx1 = [0, 1] + idx2 = [0, 0] + idx3 = [1, 1] + + out0 = x[paddle.assign(np.array(idx0))] + out1 = x[paddle.assign(np.array(idx1))] + out2 = x[paddle.assign(np.array(idx2))] + out3 = x[paddle.assign(np.array(idx3))] + + exe = paddle.static.Executor(place) + result = exe.run(prog, fetch_list=[out0, out1, out2, out3]) + + expected = [data[idx0], data[idx1], data[idx2], data[idx3]] + + self.assertTrue((result[0] == expected[0]).all()) + self.assertTrue((result[1] == expected[1]).all()) + self.assertTrue((result[2] == expected[2]).all()) + self.assertTrue((result[3] == expected[3]).all()) + + with self.assertRaises(IndexError): + one = paddle.ones(shape=[1]) + res = x[one, [0, 0]] + + def _test_slice_index_list(self, place): + data = np.random.rand(2, 3).astype("float32") + prog = paddle.static.Program() + with paddle.static.program_guard(prog): + x = paddle.assign(data) + idx0 = [1, 0] + idx1 = [0, 1] + idx2 = [0, 0] + idx3 = [1, 1] + + out0 = x[idx0] + out1 = x[idx1] + out2 = x[idx2] + out3 = x[idx3] + + exe = paddle.static.Executor(place) + result = exe.run(prog, fetch_list=[out0, out1, out2, out3]) + + expected = [data[idx0], data[idx1], data[idx2], data[idx3]] + + self.assertTrue((result[0] == expected[0]).all()) + self.assertTrue((result[1] == expected[1]).all()) + self.assertTrue((result[2] == expected[2]).all()) + self.assertTrue((result[3] == expected[3]).all()) + + def _test_slice_index_ellipsis(self, place): + data = np.random.rand(2, 3, 4).astype("float32") + prog = paddle.static.Program() + with paddle.static.program_guard(prog): + x = paddle.assign(data) + out1 = x[0:, ..., 1:] + out2 = x[0:, ...] + out3 = x[..., 1:] + out4 = x[...] + + exe = paddle.static.Executor(place) + result = exe.run(prog, fetch_list=[out1, out2, out3, out4]) + + expected = [data[0:, ..., 1:], data[0:, ...], data[..., 1:], data[...]] + + self.assertTrue((result[0] == expected[0]).all()) + self.assertTrue((result[1] == expected[1]).all()) + self.assertTrue((result[2] == expected[2]).all()) + self.assertTrue((result[3] == expected[3]).all()) + + with self.assertRaises(IndexError): + res = x[[1, 0], [0, 0]] + + with self.assertRaises(TypeError): + res = x[[1.2, 0]] + + def _test_slice_index_list_bool(self, place): + data = np.random.rand(2, 3).astype("float32") + prog = paddle.static.Program() + with paddle.static.program_guard(prog): + x = paddle.assign(data) + idx0 = [True, False] + idx1 = [False, True] + idx2 = [False, False] + idx3 = [True, True] + + out0 = x[idx0] + out1 = x[idx1] + out2 = x[idx2] + out3 = x[idx3] + + exe = paddle.static.Executor(place) + result = exe.run(prog, fetch_list=[out0, out1, out2, out3]) + + expected = [data[idx0], data[idx1], data[idx2], data[idx3]] + + self.assertTrue((result[0] == expected[0]).all()) + self.assertTrue((result[1] == expected[1]).all()) + self.assertTrue((result[2] == expected[2]).all()) + self.assertTrue((result[3] == expected[3]).all()) + + with self.assertRaises(TypeError): + res = x[[True, 0]] + def test_slice(self): + places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): - self._test_slice(core.CUDAPlace(0)) + places.append(core.CUDAPlace(0)) + + for place in places: + self._test_slice(place) + self._test_slice_index_tensor(place) + self._test_slice_index_list(place) + self._test_slice_index_ellipsis(place) + self._test_slice_index_list_bool(place) def _tostring(self): b = default_main_program().current_block() @@ -229,5 +346,61 @@ def _test(): self.assertRaises(Exception, _test) +class TestVariableSlice(unittest.TestCase): + def _test_item_none(self, place): + data = np.random.rand(2, 3, 4).astype("float32") + prog = paddle.static.Program() + with paddle.static.program_guard(prog): + x = paddle.assign(data) + out0 = x[0:, None, 1:] + out1 = x[0:, None] + out2 = x[None, 1:] + out3 = x[None] + + outs = [out0, out1, out2, out3] + exe = paddle.static.Executor(place) + result = exe.run(prog, fetch_list=outs) + + expected = [ + data[0:, None, 1:], data[0:, None], data[None, 1:], data[None] + ] + for i in range(len(outs)): + self.assertEqual(outs[i].shape, expected[i].shape) + self.assertTrue((result[i] == expected[i]).all()) + + def _test_item_none_and_decrease(self, place): + data = np.random.rand(2, 3, 4).astype("float32") + prog = paddle.static.Program() + with paddle.static.program_guard(prog): + x = paddle.assign(data) + out0 = x[0, 1:, None] + out1 = x[0, None] + out2 = x[None, 1] + out3 = x[None] + out4 = x[0, 0, 0, None] + out5 = x[None, 0, 0, 0, None] + + outs = [out0, out1, out2, out3, out4, out5] + exe = paddle.static.Executor(place) + result = exe.run(prog, fetch_list=outs) + expected = [ + data[0, 1:, None], data[0, None], data[None, 1], data[None], + data[0, 0, 0, None], data[None, 0, 0, 0, None] + ] + + for i in range(len(outs)): + self.assertEqual(outs[i].shape, expected[i].shape) + self.assertTrue((result[i] == expected[i]).all()) + + def test_slice(self): + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(core.CUDAPlace(0)) + + for place in places: + self._test_item_none(place) + self._test_item_none_and_decrease(place) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py new file mode 100644 index 0000000000000..c6ddba7feade3 --- /dev/null +++ b/python/paddle/fluid/variable_index.py @@ -0,0 +1,390 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import numpy as np +from . import unique_name +from . import core + +MAX_INTEGER = 2**31 - 1 + + +def replace_ellipsis(var, item): + from .framework import Variable + # Use slice(None) to replace Ellipsis. + # For var, var.shape = [3,4,5,6] + # + # var[..., 1:2] -> var[:, :, :, 1:2] + # var[0, ...] -> var[0] + # var[0, ..., 1:2] -> var[0, :, :, 1:2] + + item = list(item) + + # Remove Variable to skip bug when counting Ellipsis + item_remove_var = [ele for ele in item if not isinstance(ele, Variable)] + ell_count = item_remove_var.count(Ellipsis) + if ell_count == 0: + return item + elif ell_count > 1: + raise IndexError("An index can only have a single ellipsis ('...')") + + ell_idx = item.index(Ellipsis) + + if ell_idx == len(item) - 1: + return item[:-1] + else: + item[ell_idx:ell_idx + 1] = [slice(None)] * ( + len(var.shape) - len(item) + 1) + + return item + + +def replace_none(item): + new_item = [] + none_axes = [] + for i, slice_item in enumerate(item): + if slice_item is None: + none_axes.append(i) + else: + new_item.append(slice_item) + return new_item, none_axes + + +def is_integer_or_scalar_tensor(ele): + from .framework import Variable + if isinstance(ele, int): + return True + elif isinstance(ele, Variable): + if len(ele.shape) == 1 and ele.shape[0] == 1: + return True + return False + + +def deal_attrs(attrs, attr, attr_name, tensor_attr_name, inputs, infer_flags): + from .framework import Variable + from .layers import utils + + if utils._contain_var(attr): + inputs[tensor_attr_name] = utils._convert_to_tensor_list( + attr, dtype="int64") + for i, dim in enumerate(attr): + if isinstance(dim, Variable): + attrs[attr_name].append(-1) + infer_flags[i] = -1 + else: + attrs[attr_name].append(dim) + else: + attrs[attr_name] = attr + + +def _getitem_impl_(var, item): + """ + Slice the variable. + + Args: + item(int/slice/tuple) : the index. + + Returns: + Sliced variable + """ + from .framework import default_main_program, Variable + + if not isinstance(item, tuple): + item = (item, ) + + decrease_axes = [] + axes = [] + starts = [] + ends = [] + steps = [] + reverse_axes = [] + + use_strided_slice = False + item, none_axes = replace_none(item) + item = replace_ellipsis(var, item) + + for dim, slice_item in enumerate(item): + if is_integer_or_scalar_tensor(slice_item): + decrease_axes.append(dim) + start = slice_item + step = 1 + end = slice_item + 1 if slice_item != -1 else MAX_INTEGER + + elif isinstance(slice_item, slice): + start = slice_item.start + end = slice_item.stop + step = slice_item.step + + if start is None and end is None and step is None: + continue + + step = 1 if step is None else step + + if start is None and end is None: + assert (step == -1) + reverse_axes.append(dim) + continue + + start = 0 if start is None else start + end = MAX_INTEGER if end is None else end + + elif isinstance(slice_item, list): + is_bool_list = False + for i in slice_item: + if not isinstance(i, (int, bool)): + raise TypeError("Only support int or bool in index list.") + + if isinstance(i, bool): + is_bool_list = True + break + + if len(item) != 1: + raise IndexError( + "When index contains a list, its length must be 1, but received {}". + format(len(item))) + + if is_bool_list: + new_slice_item = [] + for idx, ele in enumerate(slice_item): + if not isinstance(ele, bool): + raise TypeError( + "Mixed bool index with other types is not supported." + ) + + if ele is True: + new_slice_item.append(idx) + slice_item = new_slice_item + + from .layers import assign + from ..tensor import index_select + + idx = assign(np.array(slice_item).astype("int32")) + return index_select(var, index=idx, axis=0) + + elif isinstance(slice_item, Variable): + if len(item) != 1: + raise IndexError( + "When index contains a Tensor, its length must be 1, but received {}". + format(len(item))) + + from ..tensor import index_select + return index_select(var, index=slice_item, axis=0) + + else: + raise IndexError( + "Valid index accept int or slice or ellipsis, but received {}.". + format(slice_item)) + + axes.append(dim) + starts.append(start) + ends.append(end) + steps.append(step) + use_strided_slice = True if step != 1 else use_strided_slice + + inputs = {'Input': [var]} + attrs = { + 'axes': axes, + 'starts': [], + 'ends': [], + 'decrease_axis': decrease_axes + } + if use_strided_slice: + attrs['strides'] = [] + + infer_flags = [1] * len(axes) + deal_attrs(attrs, starts, "starts", "StartsTensorList", inputs, infer_flags) + deal_attrs(attrs, ends, "ends", "EndsTensorList", inputs, infer_flags) + deal_attrs(attrs, steps, "strides", "StridesTensorList", inputs, + infer_flags) + attrs['infer_flags'] = infer_flags + + out = var + if len(axes) > 0: + target_block = default_main_program().current_block() + op_type = "strided_slice" if use_strided_slice else "slice" + + slice_out_var = target_block.create_var( + name=unique_name.generate_with_ignorable_key(var.name + "_" + + op_type), + dtype=var.dtype) + target_block.append_op( + type=op_type, + inputs=inputs, + outputs={'Out': [slice_out_var]}, + attrs=attrs) + out = slice_out_var + + if len(reverse_axes) > 0: + from .layers.tensor import reverse + out = reverse(out, axis=reverse_axes) + + # Deal with cases when all axes are decreased. + # After slice, the shape of out is [1], which should have been [], but Paddle doesn't support scalar. + # In order to ensure the correctness of the final shape of out, one dimension of out needs to be decreased. + # For example: + # # x.shape: (2,3,4) + # out = x[0, 1, 1, None] # out.shape : (1) + if len(decrease_axes) == len(var.shape): + none_axes = none_axes[1:] + + if len(none_axes) > 0: + # Deal with cases that decrease_axes is not empty + # For example: + # # x.shape: (2,3,4) + # out = x[0, 0:2, None] # out.shape : (2, 1, 4) + for idx, axis in enumerate(none_axes): + l = len([i for i in decrease_axes if i < axis]) + new_axis = axis - l + none_axes[idx] = new_axis + + # Deal with cases when all axes are decreased. + # After slice, the shape of out is [1], which should have been [], but Paddle doesn't support scalar. + # In order to ensure the correctness of the final shape of out, one dimension of out needs to be decreased. + # For example: + # # x.shape: (2,3,4) + # out = x[0, 1, 1, None] # out.shape : (1) + + from ..tensor import unsqueeze + out = unsqueeze(out, axis=none_axes) + + return out + + +def _setitem_impl_(var, item, value): + from .framework import default_main_program, Variable + + inputs = {'Input': var} + + # 1. Parse item + if not isinstance(item, tuple): + item = (item, ) + + decrease_axes = [] + axes = [] + starts = [] + ends = [] + steps = [] + + item = replace_ellipsis(var, item) + + for dim, slice_item in enumerate(item): + if is_integer_or_scalar_tensor(slice_item): + decrease_axes.append(dim) + start = slice_item + end = slice_item + 1 if slice_item != -1 else MAX_INTEGER + step = 1 + + elif isinstance(slice_item, slice): + start = slice_item.start + end = slice_item.stop + step = slice_item.step + + if start is None and end is None and step is None: + continue + + step = 1 if step is None else step + + if not isinstance(step, Variable) and step == 0: + raise ValueError( + "When assign a value to a paddle.Tensor, step can not be 0, " + "but received step is {}.".format(step)) + + if isinstance(step, Variable) and (start is None or end is None): + raise ValueError( + "When assign a value to a paddle.Tensor, it's not supported that " + "the start or end is None when the type of step is paddle.Tensor." + ) + + if start is None: + start = 0 if step > 0 else MAX_INTEGER + + if end is None: + end = MAX_INTEGER if step > 0 else (0 - MAX_INTEGER) + else: + raise IndexError( + "Valid index accept int or slice or ellipsis, but received {}.". + format(slice_item)) + + axes.append(dim) + starts.append(start) + ends.append(end) + steps.append(step) + + attrs = { + 'axes': axes, + 'starts': starts, + 'ends': ends, + 'steps': steps, + 'decrease_axes': decrease_axes + } + + from .layers import utils + if utils._contain_var(starts): + inputs['StartsTensorList'] = utils._convert_to_tensor_list(starts) + del attrs['starts'] + if utils._contain_var(ends): + inputs['EndsTensorList'] = utils._convert_to_tensor_list(ends) + del attrs['ends'] + if utils._contain_var(steps): + inputs['StepsTensorList'] = utils._convert_to_tensor_list(steps) + del attrs['steps'] + + # 2. Parse value + dtype = var.dtype + attrs['dtype'] = dtype + + from .data_feeder import convert_dtype + # 2.1 value is an integer of float + if isinstance(value, (int, float)): + value = np.array([value]).astype(convert_dtype(dtype)) + + # 2.2 value is a np.ndarray + if isinstance(value, np.ndarray): + shape = list(value.shape) + if dtype == core.VarDesc.VarType.BOOL: + value_name = "bool_values" + values = [bool(v) for v in value.flat] + elif dtype == core.VarDesc.VarType.FP32: + value_name = "fp32_values" + values = [float(v) for v in value.flat] + elif dtype == core.VarDesc.VarType.FP64: + value_name = "fp64_values" + values = [float(v) for v in value.flat] + elif dtype == core.VarDesc.VarType.INT32: + value_name = "int32_values" + values = [int(v) for v in value.flat] + elif dtype == core.VarDesc.VarType.INT64: + value_name = "int64_values" + values = [int(v) for v in value.flat] + else: + raise TypeError( + "When assign a numpy.ndarray, integer or float to a paddle.Tensor, " + "the data type of the paddle.Tensor must be bool, float32, int32 or int64, but " + "received %s." % convert_dtype(dtype)) + attrs[value_name] = values + attrs["shape"] = shape + + elif isinstance(value, Variable): + inputs["ValueTensor"] = value + else: + raise TypeError( + "Only support to assign an integer, float, numpy.ndarray or " + "paddle.Tensor to a paddle.Tensor, but received {}".format( + type(value))) + + cur_block = default_main_program().current_block() + cur_block.append_op( + type="set_value", inputs=inputs, outputs={'Out': var}, attrs=attrs) + + return var From 036f81fce9d4f732f62ec270101c87fea9882ad0 Mon Sep 17 00:00:00 2001 From: JZ-LIANG <38102074+JZ-LIANG@users.noreply.github.com> Date: Tue, 15 Jun 2021 18:13:47 +0800 Subject: [PATCH 109/156] bugfix: param init with fill constant str_value (#33381) (#33472) --- python/paddle/fluid/initializer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 5b2010f340958..54ba5f22e53d6 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -152,6 +152,7 @@ def __call__(self, var, block=None): out_dtype = var.dtype out_var = var + # fill constant should set the "str_value" to preserve precision op = block.append_op( type="fill_constant", outputs={"Out": out_var}, @@ -159,6 +160,7 @@ def __call__(self, var, block=None): "shape": var.shape, "dtype": int(out_dtype), "value": float(self._value), + 'str_value': str(float(self._value)), 'force_cpu': self._force_cpu }, stop_gradient=True) From a4e841e0d073492e3dc93abcfdce3e561df6fd32 Mon Sep 17 00:00:00 2001 From: ShenLiang Date: Tue, 15 Jun 2021 18:31:50 +0800 Subject: [PATCH 110/156] [cherry-pick] fix gather bug && fix hang of new_group (#33553) * Fix gather infer shape using axis (#33413) * fix gather shape bug * fix None * fix topo * Fix hang of hybrid parallel in new_group (#33141) * fix hang of hybrid parallel * fix new_group for hang problem * fix hang --- paddle/fluid/operators/gather.cu.h | 26 ++--- paddle/fluid/operators/gather.h | 26 ++--- paddle/fluid/operators/gather_op.cc | 33 +++++- paddle/fluid/operators/gather_op.cu | 108 +++++++----------- paddle/fluid/operators/gather_op.h | 92 +++++---------- python/paddle/distributed/collective.py | 56 +++++---- .../fluid/tests/unittests/test_gather_op.py | 1 + python/paddle/tensor/manipulation.py | 37 +++--- 8 files changed, 166 insertions(+), 213 deletions(-) diff --git a/paddle/fluid/operators/gather.cu.h b/paddle/fluid/operators/gather.cu.h index 94fe45dac0ce7..95cb428abdf34 100644 --- a/paddle/fluid/operators/gather.cu.h +++ b/paddle/fluid/operators/gather.cu.h @@ -202,12 +202,11 @@ __global__ void GatherGradGPUKernel(const T* input, const U* index, T* out, } } -template +template void GatherV2CUDAFunction(const Tensor* input, const Tensor* index, - const Tensor* axis, Tensor* out, + const int axis, Tensor* out, const paddle::platform::Place& place, const framework::ExecutionContext& ctx) { - int axis_size = axis->numel(); int index_size = index->numel(); int input_size = input->numel(); auto input_dim = input->dims(); @@ -215,12 +214,8 @@ void GatherV2CUDAFunction(const Tensor* input, const Tensor* index, auto* index_data = index->data(); if (input->numel() == 0) return; - PADDLE_ENFORCE_EQ(axis_size, 1, - platform::errors::InvalidArgument( - "Axis size should be 1, but received %d", axis_size)); - Tensor cpu_axis; - framework::TensorCopy(*axis, platform::CPUPlace(), &cpu_axis); - int axis_index = cpu_axis.data()[0]; + + int axis_index = axis; int index_dim_size = input_dim[axis_index]; int inner_dim_size = 1; @@ -251,26 +246,19 @@ void GatherV2CUDAFunction(const Tensor* input, const Tensor* index, index_size, index_dim_size, out_size); } -template +template void GatherV2GradCUDAFunction(const Tensor* input, const Tensor* index, - const Tensor* axis, Tensor* out, + const int axis, Tensor* out, const paddle::platform::Place& place, const framework::ExecutionContext& ctx) { auto* index_data = index->data(); - - int axis_size = axis->numel(); int index_size = index->numel(); int input_size = input->numel(); auto input_dim = input->dims(); auto* input_data = input->data(); if (input->numel() == 0) return; - PADDLE_ENFORCE_EQ(axis_size, 1, - platform::errors::InvalidArgument( - "Axis size should be 1, but received %d", axis_size)); - Tensor cpu_axis; - framework::TensorCopy(*axis, platform::CPUPlace(), &cpu_axis); - int axis_index = cpu_axis.data()[0]; + int axis_index = axis; int input_index_dim_size = input_dim[axis_index]; int inner_dim_size = 1; diff --git a/paddle/fluid/operators/gather.h b/paddle/fluid/operators/gather.h index c12a3b8adc978..8deab709220d7 100644 --- a/paddle/fluid/operators/gather.h +++ b/paddle/fluid/operators/gather.h @@ -126,24 +126,17 @@ void CPUGatherNd(const platform::DeviceContext& ctx, const Tensor& input, } } -template -void GatherV2Function(const Tensor* input, const Tensor* index, - const Tensor* axis, Tensor* out, - const paddle::platform::Place& place) { - auto* axis_data = axis->data(); +template +void GatherV2Function(const Tensor* input, const Tensor* index, int axis, + Tensor* out, const paddle::platform::Place& place) { auto* index_data = index->data(); - - int axis_size = axis->numel(); int index_size = index->numel(); int input_size = input->numel(); auto input_dim = input->dims(); auto* input_data = input->data(); if (input->numel() == 0) return; - PADDLE_ENFORCE_EQ(axis_size, 1, - platform::errors::InvalidArgument( - "Axis size should be 1, but received %d", axis_size)); - int axis_index = axis_data[0]; + int axis_index = axis; int input_index_dim_size = input_dim[axis_index]; for (int i = 0; i < index_size; i++) { @@ -186,22 +179,17 @@ void GatherV2Function(const Tensor* input, const Tensor* index, } } -template +template void GatherV2GradFunction(const Tensor* input, const Tensor* index, - const Tensor* axis, Tensor* out, + const int axis, Tensor* out, const paddle::platform::Place& place) { - auto* axis_data = axis->data(); auto* index_data = index->data(); - int axis_size = axis->numel(); auto input_dim = input->dims(); auto* input_data = input->data(); if (input->numel() == 0) return; - PADDLE_ENFORCE_EQ(axis_size, 1, - platform::errors::InvalidArgument( - "Axis size should be 1, but received %d", axis_size)); - int axis_index = axis_data[0]; + int axis_index = axis; int input_index_dim_size = input_dim[axis_index]; int inner_dim_size = 1; diff --git a/paddle/fluid/operators/gather_op.cc b/paddle/fluid/operators/gather_op.cc index 162766546b3c2..ea28c204ec9cf 100644 --- a/paddle/fluid/operators/gather_op.cc +++ b/paddle/fluid/operators/gather_op.cc @@ -18,6 +18,7 @@ limitations under the License. */ #include #include "paddle/fluid/framework/ddim.h" #include "paddle/fluid/framework/op_version_registry.h" + namespace paddle { namespace operators { @@ -52,11 +53,29 @@ class GatherOp : public framework::OperatorWithKernel { index_dims.size())); } - int batch_size = ctx->GetInputDim("Index")[0]; - framework::DDim output_dims(ctx->GetInputDim("X")); - output_dims[0] = batch_size; - ctx->SetOutputDim("Out", output_dims); - ctx->ShareLoD("X", /*->*/ "Out"); + auto axis = ctx->Attrs().Get("axis"); + auto input_dim = ctx->GetInputDim("X"); + if (ctx->HasInput("Axis") || axis == 0) { + // if HasInput("Axis"), we can not obtain correct shape of output + int batch_size = index_dims[0]; + framework::DDim output_dims(input_dim); + output_dims[0] = batch_size; + ctx->SetOutputDim("Out", output_dims); + ctx->ShareLoD("X", /*->*/ "Out"); + } else { + int index_size = index_dims[0]; + std::vector out_dim_vec; + for (int i = 0; i < axis; i++) { + out_dim_vec.push_back(input_dim[i]); + } + out_dim_vec.push_back(index_size); + for (int i = axis + 1; i < input_dim.size(); i++) { + out_dim_vec.push_back(input_dim[i]); + } + auto output_dims = framework::make_ddim(out_dim_vec); + ctx->SetOutputDim("Out", output_dims); + ctx->ShareLoD("X", /*->*/ "Out"); + } } protected: @@ -120,6 +139,10 @@ class GatherOpMaker : public framework::OpProtoAndCheckerMaker { "If true, update the grad using the overwrite mode in same index," "If false, using the accumulate mode in same index.") .SetDefault(true); + AddAttr( + "axis", + "The Tensor which contains the axis that we do gather operation.") + .SetDefault(0); AddComment(R"DOC( Gather Operator. diff --git a/paddle/fluid/operators/gather_op.cu b/paddle/fluid/operators/gather_op.cu index 37fbfb21f60a0..6e27d95e01855 100644 --- a/paddle/fluid/operators/gather_op.cu +++ b/paddle/fluid/operators/gather_op.cu @@ -31,47 +31,33 @@ class GatherOpCUDAKernel : public framework::OpKernel { auto *index = ctx.Input("Index"); auto *output = ctx.Output("Out"); + int axis = ctx.Attr("axis"); + + // get axis from tensor if (ctx.HasInput("Axis")) { - const Tensor *axis = ctx.Input("Axis"); - const auto &index_type = index->type(); - const auto &axis_type = axis->type(); - auto place = ctx.GetPlace(); - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT32) { - GatherV2CUDAFunction(x, index, axis, output, place, - ctx); - } - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT64) { - GatherV2CUDAFunction(x, index, axis, output, place, - ctx); + Tensor cpu_axis; + const Tensor *axis_tensor = ctx.Input("Axis"); + framework::TensorCopy(*axis_tensor, platform::CPUPlace(), &cpu_axis); + const auto &axis_type = axis_tensor->type(); + if (axis_type == framework::proto::VarType::INT32) { + axis = static_cast(cpu_axis.data()[0]); + } else if (axis_type == framework::proto::VarType::INT64) { + axis = static_cast(cpu_axis.data()[0]); } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT32) { - GatherV2CUDAFunction(x, index, axis, output, place, - ctx); - } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT64) { - GatherV2CUDAFunction(x, index, axis, output, place, - ctx); + } + const auto &place = ctx.GetPlace(); + const auto &index_type = index->type(); + if (axis != 0) { + if (index_type == framework::proto::VarType::INT32) { + GatherV2CUDAFunction(x, index, axis, output, place, ctx); + } else if (index_type == framework::proto::VarType::INT64) { + GatherV2CUDAFunction(x, index, axis, output, place, ctx); } return; } + output->mutable_data(ctx.GetPlace()); if (x->numel() == 0) return; - const auto &index_type = index->type(); - bool index_type_match = index_type == framework::proto::VarType::INT32 || - index_type == framework::proto::VarType::INT64; - PADDLE_ENFORCE_EQ(index_type_match, true, - platform::errors::InvalidArgument( - "Index holds the wrong type, it holds [%s]," - "but desires to be [%s] or [%s].", - paddle::framework::DataTypeToString(index_type), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT32), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT64))); if (index_type == framework::proto::VarType::INT32) { GPUGather(ctx.device_context(), *x, *index, output); } else if (index_type == framework::proto::VarType::INT64) { @@ -91,30 +77,27 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { auto *dX = ctx.Output(framework::GradVarName("X")); auto *dO = ctx.Input(framework::GradVarName("Out")); + int axis = ctx.Attr("axis"); if (ctx.HasInput("Axis")) { - const Tensor *axis = ctx.Input("Axis"); - const auto &index_type = index->type(); - const auto &axis_type = axis->type(); - auto place = ctx.GetPlace(); - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT32) { - GatherV2GradCUDAFunction(dO, index, axis, dX, - place, ctx); + const Tensor *axis_tensor = ctx.Input("Axis"); + Tensor cpu_axis; + framework::TensorCopy(*axis_tensor, platform::CPUPlace(), &cpu_axis); + const auto &axis_type = axis_tensor->type(); + if (axis_type == framework::proto::VarType::INT32) { + axis = static_cast(cpu_axis.data()[0]); + } else if (axis_type == framework::proto::VarType::INT64) { + axis = static_cast(cpu_axis.data()[0]); } - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT64) { - GatherV2GradCUDAFunction(dO, index, axis, dX, - place, ctx); - } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT32) { - GatherV2GradCUDAFunction(dO, index, axis, dX, - place, ctx); - } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT64) { - GatherV2GradCUDAFunction(dO, index, axis, dX, - place, ctx); + } + + const auto &index_type = index->type(); + if (axis != 0) { + if (index_type == framework::proto::VarType::INT32) { + GatherV2GradCUDAFunction(dO, index, axis, dX, + ctx.GetPlace(), ctx); + } else if (index_type == framework::proto::VarType::INT64) { + GatherV2GradCUDAFunction(dO, index, axis, dX, + ctx.GetPlace(), ctx); } return; } @@ -125,19 +108,6 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { .eigen_device(); dxt.device(place) = dxt.constant(static_cast(0)); if (dO->numel() == 0) return; - - const auto &index_type = index->type(); - bool index_type_match = index_type == framework::proto::VarType::INT32 || - index_type == framework::proto::VarType::INT64; - PADDLE_ENFORCE_EQ(index_type_match, true, - platform::errors::InvalidArgument( - "Index holds the wrong type, it holds [%s]," - "but desires to be [%s] or [%s].", - paddle::framework::DataTypeToString(index_type), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT32), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT64))); if (index_type == framework::proto::VarType::INT32) { GPUScatterAssign(ctx, *dO, *index, dX, ctx.Attr("overwrite")); diff --git a/paddle/fluid/operators/gather_op.h b/paddle/fluid/operators/gather_op.h index 8ec0d6ce0b69c..a2570c3e014e1 100644 --- a/paddle/fluid/operators/gather_op.h +++ b/paddle/fluid/operators/gather_op.h @@ -35,45 +35,30 @@ class GatherOpKernel : public framework::OpKernel { auto *index = ctx.Input("Index"); auto *output = ctx.Output("Out"); + int axis = ctx.Attr("axis"); + // get axis from tensor if (ctx.HasInput("Axis")) { - const Tensor *axis = ctx.Input("Axis"); - const auto &index_type = index->type(); - const auto &axis_type = axis->type(); - auto place = ctx.GetPlace(); - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT32) { - GatherV2Function(x, index, axis, output, place); + const Tensor *axis_tensor = ctx.Input("Axis"); + const auto &axis_type = axis_tensor->type(); + if (axis_type == framework::proto::VarType::INT32) { + axis = static_cast(axis_tensor->data()[0]); + } else if (axis_type == framework::proto::VarType::INT64) { + axis = static_cast(axis_tensor->data()[0]); } - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT64) { - GatherV2Function(x, index, axis, output, place); - } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT32) { - GatherV2Function(x, index, axis, output, place); - } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT64) { - GatherV2Function(x, index, axis, output, place); + } + const auto &place = ctx.GetPlace(); + const auto &index_type = index->type(); + if (axis != 0) { + if (index_type == framework::proto::VarType::INT32) { + GatherV2Function(x, index, axis, output, place); + } else if (index_type == framework::proto::VarType::INT64) { + GatherV2Function(x, index, axis, output, place); } return; } output->mutable_data(ctx.GetPlace()); if (x->numel() == 0) return; - - const auto &index_type = index->type(); - bool index_type_match = index_type == framework::proto::VarType::INT32 || - index_type == framework::proto::VarType::INT64; - PADDLE_ENFORCE_EQ(index_type_match, true, - platform::errors::InvalidArgument( - "Index holds the wrong type, it holds [%s]," - "but desires to be [%s] or [%s].", - paddle::framework::DataTypeToString(index_type), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT32), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT64))); if (index_type == framework::proto::VarType::INT32) { CPUGather(ctx.device_context(), *x, *index, output); } else if (index_type == framework::proto::VarType::INT64) { @@ -94,26 +79,23 @@ class GatherGradientOpKernel : public framework::OpKernel { auto *dX = ctx.Output(framework::GradVarName("X")); auto *dO = ctx.Input(framework::GradVarName("Out")); + int axis = ctx.Attr("axis"); if (ctx.HasInput("Axis")) { - const Tensor *axis = ctx.Input("Axis"); - const auto &index_type = index->type(); - const auto &axis_type = axis->type(); - auto place = ctx.GetPlace(); - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT32) { - GatherV2GradFunction(dO, index, axis, dX, place); + const Tensor *axis_tensor = ctx.Input("Axis"); + const auto &axis_type = axis_tensor->type(); + if (axis_type == framework::proto::VarType::INT32) { + axis = static_cast(axis_tensor->data()[0]); + } else if (axis_type == framework::proto::VarType::INT64) { + axis = static_cast(axis_tensor->data()[0]); } - if (index_type == framework::proto::VarType::INT32 && - axis_type == framework::proto::VarType::INT64) { - GatherV2GradFunction(dO, index, axis, dX, place); - } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT32) { - GatherV2GradFunction(dO, index, axis, dX, place); - } - if (index_type == framework::proto::VarType::INT64 && - axis_type == framework::proto::VarType::INT64) { - GatherV2GradFunction(dO, index, axis, dX, place); + } + const auto &index_type = index->type(); + + if (axis != 0) { + if (index_type == framework::proto::VarType::INT32) { + GatherV2GradFunction(dO, index, axis, dX, ctx.GetPlace()); + } else if (index_type == framework::proto::VarType::INT64) { + GatherV2GradFunction(dO, index, axis, dX, ctx.GetPlace()); } return; } @@ -126,18 +108,6 @@ class GatherGradientOpKernel : public framework::OpKernel { if (dO->numel() == 0) return; bool overwrite = ctx.Attr("overwrite"); - const auto &index_type = index->type(); - bool index_type_match = index_type == framework::proto::VarType::INT32 || - index_type == framework::proto::VarType::INT64; - PADDLE_ENFORCE_EQ(index_type_match, true, - platform::errors::InvalidArgument( - "Index holds the wrong type, it holds [%s]," - "but desires to be [%s] or [%s].", - paddle::framework::DataTypeToString(index_type), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT32), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT64))); if (index_type == framework::proto::VarType::INT32) { if (overwrite) { ScatterAssign(ctx.device_context(), *dO, *index, dX); diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index 55f86959c59f2..1a8e9a0bf55d0 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -238,31 +238,39 @@ def new_group(ranks=None, backend=None): if global_rank not in ranks: gp = Group(-1, -1, ring_id, ranks) _group_map[ring_id] = gp - return gp - - ranks = sorted(ranks) - group_rank = ranks.index(global_rank) - group_size = len(ranks) - gp = Group(group_rank, group_size, ring_id, ranks) - _group_map[ring_id] = gp - - if group_size < 2: - return gp - - strategy = core.ParallelStrategy() - strategy.nranks = group_size - strategy.local_rank = group_rank - strategy.trainer_endpoints = [genv.trainer_endpoints[i] for i in ranks] - strategy.current_endpoint = genv.current_endpoint - strategy.nrings = 1 - - if core.is_compiled_with_cuda(): - place = core.CUDAPlace(genv.device_id) - core.NCCLParallelContext(strategy, place).init_with_ring_id(ring_id) else: - assert False, ("no cuda device found") - # need to barrier to construct group - barrier(gp) + ranks = sorted(ranks) + group_rank = ranks.index(global_rank) + group_size = len(ranks) + gp = Group(group_rank, group_size, ring_id, ranks) + _group_map[ring_id] = gp + + if group_size >= 2: + strategy = core.ParallelStrategy() + strategy.nranks = group_size + strategy.local_rank = group_rank + strategy.trainer_endpoints = [ + genv.trainer_endpoints[i] for i in ranks + ] + strategy.current_endpoint = genv.current_endpoint + strategy.nrings = 1 + + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(genv.device_id) + core.NCCLParallelContext(strategy, + place).init_with_ring_id(ring_id) + else: + assert False, ("no cuda device found") + else: + return gp + + # TODO(shenliang03): This is a temporary solution to solve the problem of + # hang caused by cross-creation of new_group + tmp = paddle.to_tensor( + [1], dtype="int32") if in_dygraph_mode() else fill_constant( + [0], dtype="int32", value="1") + paddle.distributed.all_reduce(tmp, use_calc_stream=True) + paddle.distributed.wait(tmp) return gp diff --git a/python/paddle/fluid/tests/unittests/test_gather_op.py b/python/paddle/fluid/tests/unittests/test_gather_op.py index 946027a22f883..2d56441bf3eff 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_op.py @@ -182,6 +182,7 @@ def config(self): self.index_type = "int64" self.axis = [0] self.axis_type = "int32" + self.attrs = {'overwrite': False} class API_TestGather(unittest.TestCase): diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 67e6c7f8e44d7..c3031c41279c3 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -862,34 +862,39 @@ def gather(x, index, axis=None, name=None): """ if axis is None: axis = 0 - axis_tensor = axis - if not isinstance(axis, Variable) and axis == 0: - return paddle.fluid.layers.gather(input=x, index=index, overwrite=False) - if not isinstance(axis, Variable): - with device_guard("cpu"): - axis_tensor = fill_constant( - shape=[1], dtype='int64', value=axis, force_cpu=True) + if in_dygraph_mode(): - return core.ops.gather(x, index, axis_tensor) + axis = axis.item() if isinstance(axis, paddle.Tensor) else axis + return core.ops.gather(x, index, None, "axis", axis, "overwrite", False) check_variable_and_dtype( x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], 'gather') check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather') + if isinstance(axis, Variable): check_variable_and_dtype(axis, 'axis', ['int32', 'int64'], 'gather') - else: - check_type(axis, 'axis', (int), 'gather') helper = LayerHelper('gather', **locals()) dtype = helper.input_dtype('x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op( - type="gather", - inputs={"X": x, - "Index": index, - "Axis": axis_tensor}, - outputs={"Out": out}) + if not isinstance(axis, Variable): + helper.append_op( + type="gather", + inputs={"X": x, + "Index": index}, + attrs={'axis': axis, + 'overwrite': False}, + outputs={"Out": out}) + else: + helper.append_op( + type="gather", + inputs={"X": x, + "Index": index, + "Axis": axis}, + attrs={"overwrite": False}, + outputs={"Out": out}) + return out From 06c2d0c3973b76228914a6d16a497dc1a7c6a97a Mon Sep 17 00:00:00 2001 From: Peihan Date: Tue, 15 Jun 2021 19:22:33 +0800 Subject: [PATCH 111/156] [cherry-pick] tar CAPI lib in paddle build scripts (#33563) * add win_capi_tar in paddle_build.bat * tar capi lib for publish * add in gen_fluid_lib func --- paddle/scripts/paddle_build.bat | 24 +++++++++++++++++++++--- paddle/scripts/paddle_build.sh | 8 ++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index e53828ff10be6..5f157e28da6ef 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -205,7 +205,8 @@ set CUDA_ARCH_NAME=All call :cmake || goto cmake_error call :build || goto build_error -call :zip_file || goto zip_file_error +call :zip_cc_file || goto zip_cc_file_error +call :zip_c_file || goto zip_c_file_error goto:success rem "Other configurations are added here" @@ -671,7 +672,7 @@ goto:eof exit /b 1 rem --------------------------------------------------------------------------------------------- -:zip_file +:zip_cc_file tree /F %cd%\paddle_inference_install_dir\paddle if exist paddle_inference.zip del paddle_inference.zip python -c "import shutil;shutil.make_archive('paddle_inference', 'zip', root_dir='paddle_inference_install_dir')" @@ -683,10 +684,27 @@ for /F %%i in ("%libsize%") do ( ) goto:eof -:zip_file_error +:zip_cc_file_error echo Tar inference library failed! exit /b 1 +rem --------------------------------------------------------------------------------------------- +:zip_c_file +tree /F %cd%\paddle_inference_c_install_dir\paddle +if exist paddle_inference_c.zip del paddle_inference_c.zip +python -c "import shutil;shutil.make_archive('paddle_inference_c', 'zip', root_dir='paddle_inference_c_install_dir')" +%cache_dir%\tools\busybox64.exe du -h -k paddle_inference_c.zip > lib_size.txt +set /p libsize=< lib_size.txt +for /F %%i in ("%libsize%") do ( + set /a libsize_m=%%i/1024 + echo "Windows Paddle_Inference CAPI ZIP Size: !libsize_m!M" +) +goto:eof + +:zip_c_file_error +echo Tar inference capi library failed! +exit /b 1 + :timestamp setlocal enabledelayedexpansion @ECHO OFF diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 0865d48c0d343..cec7f6ef50abf 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -420,6 +420,13 @@ EOF buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/paddle_inference.tgz |awk '{print $1}') echo "Paddle_Inference Size: $buildSize" echo "ipipe_log_param_Paddle_Inference_Size: $buildSize" >> ${PADDLE_ROOT}/build/build_summary.txt + elif [ "$1" == "paddle_inference_c" ]; then + cd ${PADDLE_ROOT}/build + cp -r paddle_inference_c_install_dir paddle_inference_c + tar -czf paddle_inference_c.tgz paddle_inference_c + buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/paddle_inference_c.tgz |awk '{print $1}') + echo "Paddle_Inference Capi Size: $buildSize" + echo "ipipe_log_param_Paddle_Inference_capi_Size: $buildSize" >> ${PADDLE_ROOT}/build/build_summary.txt else SYSTEM=`uname -s` if [ "$SYSTEM" == "Darwin" ]; then @@ -1765,6 +1772,7 @@ EOF echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt build_size "paddle_inference" + build_size "paddle_inference_c" } function tar_fluid_lib() { From c334d2bd6700e9792f2f2685728d274d892f7e1c Mon Sep 17 00:00:00 2001 From: wawltor Date: Tue, 15 Jun 2021 20:38:42 +0800 Subject: [PATCH 112/156] Cherry-pick support the bool tensor for the compare ops (#33551) --- .../operators/controlflow/compare_all_op.cc | 20 ++--- .../operators/controlflow/compare_all_op.cu | 21 +++--- .../fluid/operators/controlflow/compare_op.h | 3 + .../fluid/tests/unittests/test_compare_op.py | 32 ++++++++ .../tests/unittests/test_compare_reduce_op.py | 29 +++++++- python/paddle/tensor/logic.py | 74 ++++++++++--------- 6 files changed, 126 insertions(+), 53 deletions(-) diff --git a/paddle/fluid/operators/controlflow/compare_all_op.cc b/paddle/fluid/operators/controlflow/compare_all_op.cc index adacf70f5e145..9442c7583d98f 100644 --- a/paddle/fluid/operators/controlflow/compare_all_op.cc +++ b/paddle/fluid/operators/controlflow/compare_all_op.cc @@ -135,15 +135,17 @@ class CompareReduceOp : public framework::OperatorWithKernel { ::paddle::framework::EmptyGradOpMaker, \ ::paddle::framework::EmptyGradOpMaker); -#define REGISTER_COMPARE_REDUCE_CPU_KERNEL(op_type, functor) \ - REGISTER_OP_CPU_KERNEL( \ - op_type, ::paddle::operators::CompareReduceOpKernel< \ - ::paddle::platform::CPUDeviceContext, functor>, \ - ::paddle::operators::CompareReduceOpKernel< \ - ::paddle::platform::CPUDeviceContext, functor>, \ - ::paddle::operators::CompareReduceOpKernel< \ - ::paddle::platform::CPUDeviceContext, functor>, \ - ::paddle::operators::CompareReduceOpKernel< \ +#define REGISTER_COMPARE_REDUCE_CPU_KERNEL(op_type, functor) \ + REGISTER_OP_CPU_KERNEL( \ + op_type, ::paddle::operators::CompareReduceOpKernel< \ + ::paddle::platform::CPUDeviceContext, functor>, \ + ::paddle::operators::CompareReduceOpKernel< \ + ::paddle::platform::CPUDeviceContext, functor>, \ + ::paddle::operators::CompareReduceOpKernel< \ + ::paddle::platform::CPUDeviceContext, functor>, \ + ::paddle::operators::CompareReduceOpKernel< \ + ::paddle::platform::CPUDeviceContext, functor>, \ + ::paddle::operators::CompareReduceOpKernel< \ ::paddle::platform::CPUDeviceContext, functor>); REGISTER_COMPARE_REDUCE_OP(equal_all, "X == Y"); diff --git a/paddle/fluid/operators/controlflow/compare_all_op.cu b/paddle/fluid/operators/controlflow/compare_all_op.cu index e3c920f78c45b..3753ed6b15f1e 100644 --- a/paddle/fluid/operators/controlflow/compare_all_op.cu +++ b/paddle/fluid/operators/controlflow/compare_all_op.cu @@ -85,15 +85,18 @@ class CompareReduceOpKernel } // namespace operators } // namespace paddle -#define REGISTER_COMPARE_REDUCE_CUDA_KERNEL(op_type, functor) \ - REGISTER_OP_CUDA_KERNEL( \ - op_type, paddle::operators::CompareReduceOpKernel< \ - paddle::platform::CUDADeviceContext, functor>, \ - paddle::operators::CompareReduceOpKernel< \ - paddle::platform::CUDADeviceContext, functor>, \ - paddle::operators::CompareReduceOpKernel< \ - paddle::platform::CUDADeviceContext, functor>, \ - paddle::operators::CompareReduceOpKernel< \ +#define REGISTER_COMPARE_REDUCE_CUDA_KERNEL(op_type, functor) \ + REGISTER_OP_CUDA_KERNEL( \ + op_type, paddle::operators::CompareReduceOpKernel< \ + paddle::platform::CUDADeviceContext, functor>, \ + paddle::operators::CompareReduceOpKernel< \ + paddle::platform::CUDADeviceContext, functor>, \ + paddle::operators::CompareReduceOpKernel< \ + paddle::platform::CUDADeviceContext, functor>, \ + paddle::operators::CompareReduceOpKernel< \ + paddle::platform::CUDADeviceContext, functor>, \ + paddle::operators::CompareReduceOpKernel< \ paddle::platform::CUDADeviceContext, functor>); + REGISTER_COMPARE_REDUCE_CUDA_KERNEL(equal_all, paddle::operators::EqualReduceFunctor); diff --git a/paddle/fluid/operators/controlflow/compare_op.h b/paddle/fluid/operators/controlflow/compare_op.h index ff929ee7dfce7..36185322a96b8 100644 --- a/paddle/fluid/operators/controlflow/compare_op.h +++ b/paddle/fluid/operators/controlflow/compare_op.h @@ -98,6 +98,9 @@ class CompareOpKernel #define REGISTER_COMPARE_KERNEL(op_type, dev, functor, inverse_functor) \ REGISTER_OP_##dev##_KERNEL(op_type, \ + ::paddle::operators::CompareOpKernel< \ + ::paddle::platform::dev##DeviceContext, \ + functor, inverse_functor>, \ ::paddle::operators::CompareOpKernel< \ ::paddle::platform::dev##DeviceContext, \ functor, inverse_functor>, \ diff --git a/python/paddle/fluid/tests/unittests/test_compare_op.py b/python/paddle/fluid/tests/unittests/test_compare_op.py index a2dd7e49ac4cc..7a14267588022 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_op.py @@ -155,6 +155,38 @@ def test_broadcast_api_3(self): fetch_list=[out]) self.assertEqual((res == real_result).all(), True) + def test_bool_api_4(self): + paddle.enable_static() + with program_guard(Program(), Program()): + x = paddle.static.data(name='x', shape=[3, 1], dtype='bool') + y = paddle.static.data(name='y', shape=[3, 1], dtype='bool') + op = eval("paddle.%s" % (self.op_type)) + out = op(x, y) + exe = paddle.static.Executor(self.place) + input_x = np.array([True, False, True]).astype(np.bool) + input_y = np.array([True, True, False]).astype(np.bool) + real_result = callback(input_x, input_y) + res, = exe.run(feed={"x": input_x, + "y": input_y}, + fetch_list=[out]) + self.assertEqual((res == real_result).all(), True) + + def test_bool_broadcast_api_4(self): + paddle.enable_static() + with program_guard(Program(), Program()): + x = paddle.static.data(name='x', shape=[3, 1], dtype='bool') + y = paddle.static.data(name='y', shape=[1], dtype='bool') + op = eval("paddle.%s" % (self.op_type)) + out = op(x, y) + exe = paddle.static.Executor(self.place) + input_x = np.array([True, False, True]).astype(np.bool) + input_y = np.array([True]).astype(np.bool) + real_result = callback(input_x, input_y) + res, = exe.run(feed={"x": input_x, + "y": input_y}, + fetch_list=[out]) + self.assertEqual((res == real_result).all(), True) + def test_attr_name(self): paddle.enable_static() with program_guard(Program(), Program()): diff --git a/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py b/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py index 67fe5c81ddc29..056d1687bbf84 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py @@ -92,9 +92,28 @@ def test_output(self): globals()[cls_name] = Cls +def create_test_dim1_class(op_type, typename, callback): + class Cls(op_test.OpTest): + def setUp(self): + x = y = np.random.random(size=(1)).astype(typename) + x = np.array([True, False, True]).astype(typename) + x = np.array([False, False, True]).astype(typename) + z = callback(x, y) + self.inputs = {'X': x, 'Y': y} + self.outputs = {'Out': z} + self.op_type = op_type + + def test_output(self): + self.check_output() + + cls_name = "{0}_{1}_{2}".format(op_type, typename, 'equal_all') + Cls.__name__ = cls_name + globals()[cls_name] = Cls + + np_equal = lambda _x, _y: np.array(np.array_equal(_x, _y)) -for _type_name in {'float32', 'float64', 'int32', 'int64'}: +for _type_name in {'float32', 'float64', 'int32', 'int64', 'bool'}: create_test_not_equal_class('equal_all', _type_name, np_equal) create_test_equal_class('equal_all', _type_name, np_equal) create_test_dim1_class('equal_all', _type_name, np_equal) @@ -107,6 +126,14 @@ def test_name(self): out = paddle.equal_all(x, y, name='equal_res') assert 'equal_res' in out.name + def test_dynamic_api(self): + paddle.disable_static() + x = paddle.ones(shape=[10, 10], dtype="int32") + y = paddle.ones(shape=[10, 10], dtype="int32") + out = paddle.equal_all(x, y) + assert out.numpy()[0] == True + paddle.enable_static() + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/tensor/logic.py b/python/paddle/tensor/logic.py index bdf2c477d8658..f948eeb9a48eb 100644 --- a/python/paddle/tensor/logic.py +++ b/python/paddle/tensor/logic.py @@ -38,8 +38,8 @@ def equal_all(x, y, name=None): **NOTICE**: The output of this OP has no gradient. Args: - x(Tensor): Tensor, data type is float32, float64, int32, int64. - y(Tensor): Tensor, data type is float32, float64, int32, int64. + x(Tensor): Tensor, data type is bool, float32, float64, int32, int64. + y(Tensor): Tensor, data type is bool, float32, float64, int32, int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. @@ -59,6 +59,8 @@ def equal_all(x, y, name=None): result2 = paddle.equal_all(x, z) print(result2) # result2 = [False ] """ + if in_dygraph_mode(): + return core.ops.equal_all(x, y) helper = LayerHelper("equal_all", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') @@ -152,8 +154,8 @@ def equal(x, y, name=None): **NOTICE**: The output of this OP has no gradient. Args: - x(Tensor): Tensor, data type is float32, float64, int32, int64. - y(Tensor): Tensor, data type is float32, float64, int32, int64. + x(Tensor): Tensor, data type is bool, float32, float64, int32, int64. + y(Tensor): Tensor, data type is bool, float32, float64, int32, int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. @@ -174,10 +176,10 @@ def equal(x, y, name=None): if in_dygraph_mode(): return core.ops.equal(x, y) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "equal") + check_variable_and_dtype( + x, "x", ["bool", "float32", "float64", "int32", "int64"], "equal") + check_variable_and_dtype( + y, "y", ["bool", "float32", "float64", "int32", "int64"], "equal") helper = LayerHelper("equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True @@ -196,8 +198,8 @@ def greater_equal(x, y, name=None): **NOTICE**: The output of this OP has no gradient. Args: - x(Tensor): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. - y(Tensor): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. + x(Tensor): First input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. + y(Tensor): Second input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. Returns: @@ -216,9 +218,11 @@ def greater_equal(x, y, name=None): if in_dygraph_mode(): return core.ops.greater_equal(x, y) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], + check_variable_and_dtype(x, "x", + ["bool", "float32", "float64", "int32", "int64"], "greater_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], + check_variable_and_dtype(y, "y", + ["bool", "float32", "float64", "int32", "int64"], "greater_equal") helper = LayerHelper("greater_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') @@ -240,8 +244,8 @@ def greater_than(x, y, name=None): **NOTICE**: The output of this OP has no gradient. Args: - x(Tensor): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. - y(Tensor): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. + x(Tensor): First input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. + y(Tensor): Second input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. Returns: @@ -260,9 +264,11 @@ def greater_than(x, y, name=None): if in_dygraph_mode(): return core.ops.greater_than(x, y) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], + check_variable_and_dtype(x, "x", + ["bool", "float32", "float64", "int32", "int64"], "greater_than") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], + check_variable_and_dtype(y, "y", + ["bool", "float32", "float64", "int32", "int64"], "greater_than") helper = LayerHelper("greater_than", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') @@ -284,8 +290,8 @@ def less_equal(x, y, name=None): **NOTICE**: The output of this OP has no gradient. Args: - x(Tensor): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. - y(Tensor): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. + x(Tensor): First input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. + y(Tensor): Second input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. @@ -305,10 +311,10 @@ def less_equal(x, y, name=None): if in_dygraph_mode(): return core.ops.less_equal(x, y) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "less_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "less_equal") + check_variable_and_dtype( + x, "x", ["bool", "float32", "float64", "int32", "int64"], "less_equal") + check_variable_and_dtype( + y, "y", ["bool", "float32", "float64", "int32", "int64"], "less_equal") helper = LayerHelper("less_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True @@ -327,8 +333,8 @@ def less_than(x, y, name=None): **NOTICE**: The output of this OP has no gradient. Args: - x(Tensor): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. - y(Tensor): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. + x(Tensor): First input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. + y(Tensor): Second input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. @@ -348,10 +354,10 @@ def less_than(x, y, name=None): if in_dygraph_mode(): return core.ops.less_than(x, y) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "less_than") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "less_than") + check_variable_and_dtype( + x, "x", ["bool", "float32", "float64", "int32", "int64"], "less_than") + check_variable_and_dtype( + y, "y", ["bool", "float32", "float64", "int32", "int64"], "less_than") helper = LayerHelper("less_than", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True @@ -370,8 +376,8 @@ def not_equal(x, y, name=None): **NOTICE**: The output of this OP has no gradient. Args: - x(Tensor): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. - y(Tensor): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. + x(Tensor): First input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. + y(Tensor): Second input to compare which is N-D tensor. The input data type should be bool, float32, float64, int32, int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. @@ -391,10 +397,10 @@ def not_equal(x, y, name=None): if in_dygraph_mode(): return core.ops.not_equal(x, y) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "not_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "not_equal") + check_variable_and_dtype( + x, "x", ["bool", "float32", "float64", "int32", "int64"], "not_equal") + check_variable_and_dtype( + y, "y", ["bool", "float32", "float64", "int32", "int64"], "not_equal") helper = LayerHelper("not_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True From e5bd7eb82eca1eeb83a742e48eea0dd1d284fbab Mon Sep 17 00:00:00 2001 From: Shang Zhizhou Date: Wed, 16 Jun 2021 10:29:03 +0800 Subject: [PATCH 113/156] Add trt layer norm dynamic (#33448) * 1, remove layernorm dynamic fp16; 2, let reshape out in dynamic shape (#33535) --- .../tensorrt/convert/layer_norm_op.cc | 38 +++-- paddle/fluid/inference/tensorrt/op_teller.cc | 2 +- .../tensorrt/plugin/layer_norm_op_plugin.cu | 109 ++++++++++++- .../tensorrt/plugin/layer_norm_op_plugin.h | 149 +++++++++++++++++- paddle/fluid/pybind/inference_api.cc | 1 + .../ir/inference/inference_pass_test.py | 5 +- .../ir/inference/test_trt_subgraph_pass.py | 55 +++++++ 7 files changed, 336 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc b/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc index 0b97b5d87a3d5..de5d3110e1890 100644 --- a/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc @@ -46,13 +46,6 @@ class LayerNormOpConverter : public OpConverter { auto* Bias_t = Bias_v->GetMutable(); auto* Scale_t = Scale_v->GetMutable(); - int input_num = 1; - for (int i = 0; i < X->getDimensions().nbDims; i++) { - input_num *= X->getDimensions().d[i]; - } - std::vector mean_shape{input_num}; - std::vector variance_shape{input_num}; - std::unique_ptr bias_tensor( new framework::LoDTensor()); std::unique_ptr scale_tensor( @@ -68,10 +61,33 @@ class LayerNormOpConverter : public OpConverter { auto* bias_data = bias_tensor->mutable_data(platform::CPUPlace()); auto* scale_data = scale_tensor->mutable_data(platform::CPUPlace()); - plugin::LayerNormPlugin* plugin = new plugin::LayerNormPlugin( - bias_data, bias_tensor->numel(), scale_data, scale_tensor->numel(), - begin_norm_axis, eps, mean_shape, variance_shape); - nvinfer1::IPluginLayer* layernorm_layer = engine_->AddPlugin(&X, 1, plugin); + nvinfer1::ILayer* layernorm_layer = nullptr; + if (engine_->with_dynamic_shape()) { + int input_num = 1; + for (int i = begin_norm_axis; i < X->getDimensions().nbDims; i++) { + input_num *= X->getDimensions().d[i]; + } + std::vector mean_shape{input_num}; + std::vector variance_shape{input_num}; + plugin::LayerNormPluginDynamic* plugin = + new plugin::LayerNormPluginDynamic(bias_data, bias_tensor->numel(), + scale_data, scale_tensor->numel(), + begin_norm_axis, eps, mean_shape, + variance_shape); + layernorm_layer = engine_->AddDynamicPlugin(&X, 1, plugin); + } else { + int input_num = 1; + for (int i = begin_norm_axis - 1; i < X->getDimensions().nbDims; i++) { + input_num *= X->getDimensions().d[i]; + } + std::vector mean_shape{input_num}; + std::vector variance_shape{input_num}; + plugin::LayerNormPlugin* plugin = new plugin::LayerNormPlugin( + bias_data, bias_tensor->numel(), scale_data, scale_tensor->numel(), + begin_norm_axis, eps, mean_shape, variance_shape); + layernorm_layer = engine_->AddPlugin( + &X, 1, reinterpret_cast(plugin)); + } auto output_name = op_desc.Output("Y").front(); engine_->SetWeights(op_desc.Input("Bias").front(), std::move(bias_tensor)); diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 07dc1a0684e8e..44611d1d5959d 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -700,7 +700,7 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } if (op_type == "reshape" || op_type == "reshape2") { - if (!desc.HasAttr("shape") || with_dynamic_shape) { + if (!desc.HasAttr("shape")) { return false; // Paddle-TRT does not support the input tensors: Shape and ShapeTensor } else if (desc.Input("Shape").size() >= 1 || diff --git a/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.cu index 8af036a0e8670..f9341613a0f55 100644 --- a/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.cu @@ -57,8 +57,18 @@ int LayerNormPlugin::enqueue(int batch_size, const void *const *inputs, input_shape.push_back(input_dims.d[i]); } const auto input_ddim = framework::make_ddim(input_shape); - auto matrix_dim = framework::flatten_to_2d(input_ddim, begin_norm_axis - 1); + auto matrix_dim = framework::flatten_to_2d(input_ddim, begin_norm_axis); int feature_size = static_cast(matrix_dim[1]); + PADDLE_ENFORCE_EQ(feature_size, scale_.size(), + platform::errors::InvalidArgument( + "scale's size should be equal to the feature_size," + "but got feature_size:%d, scale's size:%d.", + feature_size, scale_.size())); + PADDLE_ENFORCE_EQ(feature_size, bias_.size(), + platform::errors::InvalidArgument( + "bias's size should be equal to the feature_size," + "but got feature_size:%d, bias's size:%d.", + feature_size, bias_.size())); scale_t.Resize(framework::make_ddim({feature_size})); bias_t.Resize(framework::make_ddim({feature_size})); @@ -82,6 +92,103 @@ int LayerNormPlugin::enqueue(int batch_size, const void *const *inputs, return cudaGetLastError() != cudaSuccess; } +nvinfer1::DimsExprs LayerNormPluginDynamic::getOutputDimensions( + int output_index, const nvinfer1::DimsExprs *inputDims, int nb_inputs, + nvinfer1::IExprBuilder &expr_builder) { + return inputDims[0]; +} + +bool LayerNormPluginDynamic::supportsFormatCombination( + int pos, const nvinfer1::PluginTensorDesc *in_out, int nb_inputs, + int nb_outputs) { + PADDLE_ENFORCE_NOT_NULL( + in_out, platform::errors::InvalidArgument( + "The input of layernorm plugin shoule not be nullptr.")); + PADDLE_ENFORCE_LT( + pos, nb_inputs + nb_outputs, + platform::errors::InvalidArgument("The pos(%d) should be less than the " + "num(%d) of the input and the output.", + pos, nb_inputs + nb_outputs)); + const nvinfer1::PluginTensorDesc &in = in_out[pos]; + if (pos == 0) { + // TODO(Shangzhizhou) FP16 support + return (in.type == nvinfer1::DataType::kFLOAT) && + (in.format == nvinfer1::TensorFormat::kLINEAR); + } + const nvinfer1::PluginTensorDesc &prev = in_out[pos - 1]; + // output + return in.type == prev.type && in.format == prev.format; +} + +nvinfer1::DataType LayerNormPluginDynamic::getOutputDataType( + int index, const nvinfer1::DataType *input_types, int nb_inputs) const { + PADDLE_ENFORCE_EQ(index, 0, + platform::errors::InvalidArgument( + "The LayerNormPlugin only has one input, so the " + "index value should be 0, but get %d.", + index)); + return input_types[0]; +} + +int LayerNormPluginDynamic::enqueue( + const nvinfer1::PluginTensorDesc *input_desc, + const nvinfer1::PluginTensorDesc *output_desc, const void *const *inputs, + void *const *outputs, void *workspace, cudaStream_t stream) { + const auto &input_dims = input_desc[0].dims; + int begin_norm_axis = begin_norm_axis_; + float eps = eps_; + + std::vector input_shape; + for (int i = 0; i < input_dims.nbDims; i++) { + input_shape.push_back(input_dims.d[i]); + } + const auto input_ddim = framework::make_ddim(input_shape); + auto matrix_dim = framework::flatten_to_2d(input_ddim, begin_norm_axis); + int feature_size = static_cast(matrix_dim[1]); + PADDLE_ENFORCE_EQ(feature_size, scale_.size(), + platform::errors::InvalidArgument( + "scale's size should be equal to the feature_size," + "but got feature_size:%d, scale's size:%d.", + feature_size, scale_.size())); + PADDLE_ENFORCE_EQ(feature_size, bias_.size(), + platform::errors::InvalidArgument( + "bias's size should be equal to the feature_size," + "but got feature_size:%d, bias's size:%d.", + feature_size, bias_.size())); + int device_id; + cudaGetDevice(&device_id); + auto input_type = input_desc[0].type; + if (input_type == nvinfer1::DataType::kFLOAT) { + VLOG(1) << "TRT Plugin DataType selected. LayerNorm-->fp32"; + const float *input = reinterpret_cast(inputs[0]); + float *output = static_cast(outputs[0]); + scale_t.Resize(framework::make_ddim({feature_size})); + bias_t.Resize(framework::make_ddim({feature_size})); + mean_t.Resize(framework::make_ddim(mean_shape_)); + variance_t.Resize(framework::make_ddim(variance_shape_)); + + float *scale_d = + scale_t.mutable_data(platform::CUDAPlace(device_id)); + float *bias_d = bias_t.mutable_data(platform::CUDAPlace(device_id)); + float *mean_d = mean_t.mutable_data(platform::CUDAPlace(device_id)); + float *variance_d = + variance_t.mutable_data(platform::CUDAPlace(device_id)); + + cudaMemcpyAsync(scale_d, scale_.data(), sizeof(float) * feature_size, + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(bias_d, bias_.data(), sizeof(float) * feature_size, + cudaMemcpyHostToDevice, stream); + + paddle::operators::LayerNormDirectCUDAFunctor layer_norm; + layer_norm(stream, input, input_shape, bias_d, scale_d, output, mean_d, + variance_d, begin_norm_axis, eps); + } else { + PADDLE_THROW(platform::errors::Fatal( + "The LayerNorm TRT Plugin's input type should be float.")); + } + return cudaGetLastError() != cudaSuccess; +} + } // namespace plugin } // namespace tensorrt } // namespace inference diff --git a/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.h b/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.h index 050ef3b77d315..9c4c31b61e128 100644 --- a/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/layer_norm_op_plugin.h @@ -50,7 +50,7 @@ class LayerNormPlugin : public PluginTensorRT { // TRT will call this func when we need to serialize the configuration of // tensorrt. // It should not be called by users. - void serialize(void *buffer) override { + void serialize(void* buffer) override { SerializeValue(&buffer, getPluginType()); serializeBase(buffer); SerializeValue(&buffer, bias_); @@ -62,7 +62,7 @@ class LayerNormPlugin : public PluginTensorRT { } public: - LayerNormPlugin(const float *bias, const int bias_num, const float *scale, + LayerNormPlugin(const float* bias, const int bias_num, const float* scale, const int scale_num, int begin_norm_axis, float eps, std::vector mean_shape, std::vector variance_shape) @@ -78,7 +78,7 @@ class LayerNormPlugin : public PluginTensorRT { // It was used for tensorrt deserialization. // It should not be called by users. - LayerNormPlugin(void const *serialData, size_t serialLength) { + LayerNormPlugin(void const* serialData, size_t serialLength) { deserializeBase(serialData, serialLength); DeserializeValue(&serialData, &serialLength, &bias_); DeserializeValue(&serialData, &serialLength, &scale_); @@ -90,20 +90,153 @@ class LayerNormPlugin : public PluginTensorRT { ~LayerNormPlugin() {} int initialize() override; - LayerNormPlugin *clone() const override { + LayerNormPlugin* clone() const override { return new LayerNormPlugin(bias_.data(), bias_.size(), scale_.data(), scale_.size(), begin_norm_axis_, eps_, mean_shape_, variance_shape_); } - const char *getPluginType() const override { return "layer_norm_plugin"; } + const char* getPluginType() const override { return "layer_norm_plugin"; } int getNbOutputs() const override { return 1; } - nvinfer1::Dims getOutputDimensions(int index, const nvinfer1::Dims *inputs, + nvinfer1::Dims getOutputDimensions(int index, const nvinfer1::Dims* inputs, int nbInputDims) override; - int enqueue(int batchSize, const void *const *inputs, void **outputs, - void *workspace, cudaStream_t stream) override; + int enqueue(int batchSize, const void* const* inputs, void** outputs, + void* workspace, cudaStream_t stream) override; }; +class LayerNormPluginDynamic : public DynamicPluginTensorRT { + public: + LayerNormPluginDynamic(const float* bias, const int bias_num, + const float* scale, const int scale_num, + int begin_norm_axis, float eps, + std::vector mean_shape, + std::vector variance_shape) + : begin_norm_axis_(begin_norm_axis), + eps_(eps), + mean_shape_(mean_shape), + variance_shape_(variance_shape) { + bias_.resize(bias_num); + scale_.resize(scale_num); + std::copy(bias, bias + bias_num, bias_.data()); + std::copy(scale, scale + scale_num, scale_.data()); + } + + LayerNormPluginDynamic(void const* serialData, size_t serialLength) { + DeserializeValue(&serialData, &serialLength, &bias_); + DeserializeValue(&serialData, &serialLength, &scale_); + DeserializeValue(&serialData, &serialLength, &begin_norm_axis_); + DeserializeValue(&serialData, &serialLength, &eps_); + DeserializeValue(&serialData, &serialLength, &mean_shape_); + DeserializeValue(&serialData, &serialLength, &variance_shape_); + } + nvinfer1::IPluginV2DynamicExt* clone() const override { + return new LayerNormPluginDynamic(bias_.data(), bias_.size(), scale_.data(), + scale_.size(), begin_norm_axis_, eps_, + mean_shape_, variance_shape_); + } + + const char* getPluginType() const override { return "layernorm_plugin"; } + int getNbOutputs() const override { return 1; } + int initialize() override { return 0; } + + size_t getSerializationSize() const override { + return SerializedSize(bias_) + SerializedSize(scale_) + + SerializedSize(begin_norm_axis_) + SerializedSize(eps_) + + SerializedSize(mean_shape_) + SerializedSize(variance_shape_); + } + + void serialize(void* buffer) const override { + SerializeValue(&buffer, bias_); + SerializeValue(&buffer, scale_); + SerializeValue(&buffer, begin_norm_axis_); + SerializeValue(&buffer, eps_); + SerializeValue(&buffer, mean_shape_); + SerializeValue(&buffer, variance_shape_); + } + + nvinfer1::DimsExprs getOutputDimensions( + int output_index, const nvinfer1::DimsExprs* inputs, int nb_inputs, + nvinfer1::IExprBuilder& expr_builder) override; + + bool supportsFormatCombination(int pos, + const nvinfer1::PluginTensorDesc* inOut, + int nbInputs, int nbOutputs) override; + + void configurePlugin(const nvinfer1::DynamicPluginTensorDesc* in, + int nbInputs, + const nvinfer1::DynamicPluginTensorDesc* out, + int nbOutputs) override {} + + size_t getWorkspaceSize(const nvinfer1::PluginTensorDesc* inputs, + int nbInputs, + const nvinfer1::PluginTensorDesc* outputs, + int nbOutputs) const override { + return 0; + } + + int enqueue(const nvinfer1::PluginTensorDesc* inputDesc, + const nvinfer1::PluginTensorDesc* outputDesc, + const void* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) override; + nvinfer1::DataType getOutputDataType(int index, + const nvinfer1::DataType* inputTypes, + int nbInputs) const override; + + void destroy() override { delete this; } + + private: + std::vector bias_; + std::vector scale_; + framework::Tensor scale_t; + framework::Tensor bias_t; + framework::Tensor mean_t; + framework::Tensor variance_t; + int begin_norm_axis_; + float eps_; + std::vector mean_shape_; + std::vector variance_shape_; +}; + +class LayerNormPluginDynamicCreator : public nvinfer1::IPluginCreator { + public: + LayerNormPluginDynamicCreator() {} + const char* getPluginName() const override { return "layernorm_plugin"; } + + const char* getPluginVersion() const override { return "1"; } + + const nvinfer1::PluginFieldCollection* getFieldNames() override { + return &field_collection_; + } + + nvinfer1::IPluginV2* createPlugin( + const char* name, const nvinfer1::PluginFieldCollection* fc) override { + return nullptr; + } + + nvinfer1::IPluginV2* deserializePlugin(const char* name, + const void* serial_data, + size_t serial_length) override { + auto plugin = new LayerNormPluginDynamic(serial_data, serial_length); + return plugin; + } + + void setPluginNamespace(const char* lib_namespace) override { + plugin_namespace_ = lib_namespace; + } + + const char* getPluginNamespace() const override { + return plugin_namespace_.c_str(); + } + + private: + std::string plugin_namespace_; + std::string plugin_name_; + nvinfer1::PluginFieldCollection field_collection_{0, nullptr}; + std::vector plugin_attributes_; +}; + +REGISTER_TRT_PLUGIN_V2(LayerNormPluginDynamicCreator); + } // namespace plugin } // namespace tensorrt } // namespace inference diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc index 8a5ad5852aedf..b2572e5aa4ba1 100644 --- a/paddle/fluid/pybind/inference_api.cc +++ b/paddle/fluid/pybind/inference_api.cc @@ -511,6 +511,7 @@ void BindAnalysisConfig(py::module *m) { py::arg("disable_trt_plugin_fp16") = false) .def("enable_tensorrt_oss", &AnalysisConfig::EnableTensorRtOSS) .def("tensorrt_oss_enabled", &AnalysisConfig::tensorrt_oss_enabled) + .def("exp_disable_tensorrt_ops", &AnalysisConfig::Exp_DisableTensorRtOPs) .def("enable_tensorrt_dla", &AnalysisConfig::EnableTensorRtDLA, py::arg("dla_core") = 0) .def("tensorrt_dla_enabled", &AnalysisConfig::tensorrt_dla_enabled) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py b/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py index 010086bfbbc47..e3c21eaa78d71 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py @@ -160,7 +160,8 @@ def check_output_with_option(self, use_gpu, atol=1e-5, flatten=False, - quant=False): + quant=False, + rtol=1e-5): ''' Check whether calculating on CPU and GPU, enable TensorRT or disable TensorRT, enable MKLDNN or disable MKLDNN @@ -260,7 +261,7 @@ def check_output_with_option(self, self.assertTrue( np.allclose( - out, tensorrt_output, atol=atol), + out, tensorrt_output, rtol=rtol, atol=atol), "Output has diff between GPU and TensorRT. ") # Check whether the mkldnn results and the CPU results are the same. diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py index bdcdeee8dcb66..25d0173ef5ead 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py @@ -367,6 +367,61 @@ def test_check_output(self): PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) +class TensorRTSubgraphPassLayerNormDynamicTest(InferencePassTest): + def setUp(self): + self.set_params() + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32") + out = fluid.layers.layer_norm( + data, begin_norm_axis=self.begin_norm_axis) + self.feeds = { + "data": np.random.random([1, 3, 64, 64]).astype("float32"), + } + self.set_trt_params() + self.fetch_list = [out] + + def set_trt_params(self): + self.enable_trt = True + self.trt_parameters = TensorRTSubgraphPassLayerNormDynamicTest.TensorRTParam( + 1 << 30, 32, 0, self.precision, self.serialize, False) + self.dynamic_shape_params = TensorRTSubgraphPassLayerNormDynamicTest.DynamicShapeParam( + { + 'data': [1, 3, 64, 64], + }, {'data': [8, 8, 64, 64], }, {'data': [4, 4, 64, 64], }, False) + + def set_params(self): + self.begin_norm_axis = 2 + self.precision = AnalysisConfig.Precision.Float32 + self.serialize = True + + def test_check_output(self): + if os.path.exists(self.path + "_opt_cache"): + shutil.rmtree(self.path + "_opt_cache") + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + +class TensorRTSubgraphPassLayerNormDynamicFP16Test( + TensorRTSubgraphPassLayerNormDynamicTest): + def set_params(self): + self.begin_norm_axis = 2 + self.precision = AnalysisConfig.Precision.Half + self.serialize = True + + def test_check_output(self): + if os.path.exists(self.path + "_opt_cache"): + shutil.rmtree(self.path + "_opt_cache") + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu, atol=0.01, rtol=0.01) + self.assertTrue( + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + + class TensorRTSubgraphPassLayerNormBeginNormAxis2Test( TensorRTSubgraphPassLayerNormTest): def set_params(self): From 5c68e79d78372b73ad9b74fe1b32259da577355c Mon Sep 17 00:00:00 2001 From: lidanqing Date: Wed, 16 Jun 2021 10:31:23 +0800 Subject: [PATCH 114/156] [cherry pick] Fix issue #33021 setCacheCapacity could not limit memory consumption (#33571) * [oneDNN] First fix to #33021 (#33174) * - First fix to #33021 * [oneDNN] Second fix to #33021 (#33471) * use older download_data function Co-authored-by: Jacek Czaja --- .../fluid/inference/api/analysis_predictor.cc | 12 +- .../fluid/inference/tests/api/CMakeLists.txt | 9 +- ...nalyzer_detect_functional_mkldnn_tester.cc | 166 ++++++++++++++++++ paddle/fluid/platform/device_context.cc | 31 +++- paddle/fluid/platform/device_context.h | 15 +- 5 files changed, 212 insertions(+), 21 deletions(-) create mode 100644 paddle/fluid/inference/tests/api/analyzer_detect_functional_mkldnn_tester.cc diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 42793595e19c8..215174c12ce3b 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -343,8 +343,6 @@ void AnalysisPredictor::MkldnnPreSet( platform::MKLDNNDeviceContext::tls().set_cur_mkldnn_session_id( platform::MKLDNNDeviceContextThreadLocals:: kMKLDNNSessionID_CacheClearing); - platform::MKLDNNDeviceContext::tls().set_cur_input_shape_cache_capacity( - config_.mkldnn_cache_capacity_); // Set current_input_shape for caching dynamic shape. std::stringstream ss; for (size_t i = 0; i < inputs_shape.size(); ++i) { @@ -355,6 +353,9 @@ void AnalysisPredictor::MkldnnPreSet( VLOG(2) << "Set input shape=" << ss.str(); platform::MKLDNNDeviceContext::tls().set_cur_input_shape_str(ss.str()); } + platform::MKLDNNDeviceContext::tls().set_cur_input_shape_cache_capacity( + config_.mkldnn_cache_capacity_); + #endif } @@ -370,10 +371,9 @@ void AnalysisPredictor::MkldnnPostReset() { CHECK_LE(shape_blob_size, static_cast(config_.mkldnn_cache_capacity_)); } - paddle::platform::MKLDNNDeviceContext::tls().set_cur_mkldnn_session_id( - platform::MKLDNNDeviceContextThreadLocals::kMKLDNNSessionID_Default); - platform::MKLDNNDeviceContext::tls().set_cur_input_shape_cache_capacity(0); - platform::MKLDNNDeviceContext::tls().set_cur_input_shape_str(""); + // We cannot reset to the default cache settings + // as there maybe CopyToCPU method used and oneDNN + // primitives are used there so cache would grow } #endif } diff --git a/paddle/fluid/inference/tests/api/CMakeLists.txt b/paddle/fluid/inference/tests/api/CMakeLists.txt index f74cd671d6dca..0df442d332cd8 100644 --- a/paddle/fluid/inference/tests/api/CMakeLists.txt +++ b/paddle/fluid/inference/tests/api/CMakeLists.txt @@ -285,11 +285,10 @@ inference_analysis_api_test(test_analyzer_ocr ${OCR_INSTALL_DIR} analyzer_vis_te # densebox set(DENSEBOX_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/densebox") download_data(${DENSEBOX_INSTALL_DIR} "densebox.tar.gz") -#inference_analysis_test(test_analyzer_detect SRCS analyzer_detect_tester.cc -# EXTRA_DEPS ${INFERENCE_EXTRA_DEPS} -# ARGS --infer_model=${DENSEBOX_INSTALL_DIR}/model --infer_data=${DENSEBOX_INSTALL_DIR}/detect_input_50.txt -# --infer_shape=${DENSEBOX_INSTALL_DIR}/shape_50.txt) -#set_property(TEST test_analyzer_detect PROPERTY ENVIRONMENT GLOG_vmodule=analysis_predictor=2) +inference_analysis_test(test_analyzer_detect_functional_mkldnn SRCS analyzer_detect_functional_mkldnn_tester.cc + EXTRA_DEPS ${INFERENCE_EXTRA_DEPS} + ARGS --infer_model=${DENSEBOX_INSTALL_DIR}/model --infer_data=${DENSEBOX_INSTALL_DIR}/detect_input_50.txt + --infer_shape=${DENSEBOX_INSTALL_DIR}/shape_50.txt) # mobilenet with transpose op set(MOBILENET_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/mobilenet") diff --git a/paddle/fluid/inference/tests/api/analyzer_detect_functional_mkldnn_tester.cc b/paddle/fluid/inference/tests/api/analyzer_detect_functional_mkldnn_tester.cc new file mode 100644 index 0000000000000..384bef8a4b439 --- /dev/null +++ b/paddle/fluid/inference/tests/api/analyzer_detect_functional_mkldnn_tester.cc @@ -0,0 +1,166 @@ +/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/fluid/inference/tests/api/tester_helper.h" +#include "paddle/fluid/platform/device_context.h" +#include "paddle/fluid/platform/place.h" + +DEFINE_string(infer_shape, "", "data shape file"); +DEFINE_int32(sample, 20, "number of sample"); + +namespace paddle { +namespace inference { +namespace analysis { + +struct Record { + std::vector data; + std::vector shape; +}; + +Record ProcessALine(const std::string &line, const std::string &shape_line) { + VLOG(3) << "process a line"; + + Record record; + std::vector data_strs; + split(line, ' ', &data_strs); + for (auto &d : data_strs) { + record.data.push_back(std::stof(d)); + } + + std::vector shape_strs; + split(shape_line, ' ', &shape_strs); + for (auto &s : shape_strs) { + record.shape.push_back(std::stoi(s)); + } + return record; +} + +void SetConfig(AnalysisConfig *cfg) { + cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params"); + cfg->DisableGpu(); + // cfg->SwitchIrDebug(); // Enable to have graphs dumped + cfg->SwitchSpecifyInputNames(false); + cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads); +} + +void SetInput(std::vector> *inputs, + const std::string &line, const std::string &shape_line) { + auto record = ProcessALine(line, shape_line); + + PaddleTensor input; + input.shape = record.shape; + input.dtype = PaddleDType::FLOAT32; + size_t input_size = record.data.size() * sizeof(float); + input.data.Resize(input_size); + memcpy(input.data.data(), record.data.data(), input_size); + std::vector input_slots; + input_slots.assign({input}); + (*inputs).emplace_back(input_slots); +} + +#ifdef PADDLE_WITH_MKLDNN +int GetNumCachedObjects(void) { + auto &pool = platform::DeviceContextPool::Instance(); + platform::CPUPlace place; + auto onednn_dev_ctx = + dynamic_cast(pool.Get(place)); + return onednn_dev_ctx->GetCachedObjectsNumber(); +} + +void validate_cache_onednn(int cache_capacity = 1) { + AnalysisConfig cfg; + SetConfig(&cfg); + cfg.EnableMKLDNN(); + cfg.SetMkldnnCacheCapacity(cache_capacity); + + auto predictor = CreatePaddlePredictor(cfg); + std::vector> ref_outputs; + std::vector> input_slots_all; + + std::ifstream file(FLAGS_infer_data); + std::ifstream infer_file(FLAGS_infer_shape); + std::vector lines; + std::vector shape_lines; + + // Let's work with 4 samples + auto num_samples = 4; + ref_outputs.resize(num_samples); + lines.resize(num_samples); + shape_lines.resize(num_samples); + + // Let's remember number of cached objects before + // execution and after every single execution + std::vector cache_filling; + cache_filling.push_back(GetNumCachedObjects()); + + // compute sequentially prediction + for (int i = 0; i < num_samples; ++i) { + std::getline(file, lines[i]); + std::getline(infer_file, shape_lines[i]); + SetInput(&input_slots_all, lines[i], shape_lines[i]); + predictor->Run(input_slots_all[i], &ref_outputs[i], FLAGS_batch_size); + // record number of cached objects + cache_filling.push_back(GetNumCachedObjects()); + } + + file.close(); + infer_file.close(); + + // Pick first output tensor from model + // as internally reorders may be called + // so it will impact cache size + auto output_names = predictor->GetOutputNames(); + auto output_t = predictor->GetOutputTensor(output_names[0]); + std::vector output_shape = output_t->shape(); + size_t out_num = std::accumulate(output_shape.begin(), output_shape.end(), 1, + std::multiplies()); + std::vector out_data; + out_data.resize(out_num); + output_t->CopyToCpu(out_data.data()); + + // Release predictor (relevant cache should be emptied) + predictor.reset(nullptr); + cache_filling.push_back(GetNumCachedObjects()); + + // Compare results + // First and last value should be equal e.g. before using cache (empty) and + // after releasing executor + PADDLE_ENFORCE_EQ( + cache_filling[0], cache_filling[cache_filling.size() - 1], + platform::errors::Fatal("Cache size before execution and after " + "releasing Executor do not match")); + + // Iterate to check if cache is not increasing + // over exceeding cache capacity + if (cache_capacity != 0) { + for (int i = cache_capacity + 1; i < num_samples + 1; ++i) { + PADDLE_ENFORCE_EQ( + cache_filling[cache_capacity], cache_filling[i], + platform::errors::Fatal("Cache capacity should not increase " + "after full capacity is used")); + } + } +} + +TEST(Analyzer_detect, validate_cache_onednn) { + validate_cache_onednn(2 /*cache_capacity */); +} +#endif + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/platform/device_context.cc b/paddle/fluid/platform/device_context.cc index 9a47ac45462ed..fcb60b27b19d5 100644 --- a/paddle/fluid/platform/device_context.cc +++ b/paddle/fluid/platform/device_context.cc @@ -537,7 +537,7 @@ Place CUDAPinnedDeviceContext::GetPlace() const { return place_; } MKLDNNDeviceContext::MKLDNNDeviceContext(CPUPlace place) : CPUDeviceContext(place), p_blobmap_() { p_blobmap_.reset(new BlobMap()); - p_exec_items_.reset(new ExecMap()); + p_exec_items_.reset(new ExecShape()); p_mutex_.reset(new std::mutex()); } @@ -618,10 +618,15 @@ void MKLDNNDeviceContext::ResetBlobMap(void* ptr) { if (ptr == nullptr) { p_blobmap_->clear(); } else { - for (auto& v : (*p_exec_items_)[ptr]) { - (v.first)->erase(v.second); + // Iterate through all shapes and release + // for each shape and active executor all entries + // of this executor + for (auto& s : *p_exec_items_) { + for (auto& v : (*s.second)[ptr]) { + (v.first)->erase(v.second); + } + s.second->erase(ptr); } - p_exec_items_->erase(ptr); } } else { VLOG(3) << "Prevented Clearing DNNL cache."; @@ -629,11 +634,24 @@ void MKLDNNDeviceContext::ResetBlobMap(void* ptr) { } } +void MKLDNNDeviceContext::RemoveShapeEntriesWithExecutor(void) const { + p_exec_items_->erase(p_exec_items_->begin()); +} + void MKLDNNDeviceContext::LinkEntryWithExecutor(BlobPtr_t pblob, KeyBlob::iterator it) const { + // Take current input shape from TLS // Take current executor addess from TLS // and for this executor's items add the one defined with arguments - (*p_exec_items_)[tls().get_curr_exec()].push_back(std::make_pair(pblob, it)); + auto key_it = p_exec_items_ + ->insert(std::make_pair(tls().cur_input_shape_str, + std::make_shared())) + .first; + (*key_it->second)[tls().get_curr_exec()].push_back(std::make_pair(pblob, it)); + + VLOG(3) << "LinkEntryWithExecutor, shapes: " << p_exec_items_->size() + << " curr exec size: " + << (*key_it->second)[tls().get_curr_exec()].size() << "\n"; } void MKLDNNDeviceContext::BlockNextCacheClearing() { @@ -690,6 +708,7 @@ void MKLDNNDeviceContext::SetBlob(const std::string& name, VLOG(2) << "sid=" << sid << ", remove all blobs of shape: " << sBlob->begin()->first; sBlob->erase(sBlob->begin()->first); + RemoveShapeEntriesWithExecutor(); } pBlob = std::make_shared(); (*sBlob)[tls().cur_input_shape_str] = pBlob; @@ -713,7 +732,7 @@ void MKLDNNDeviceContext::SetBlob(const std::string& name, return; } -unsigned int MKLDNNDeviceContext::GetCachedObjectsNumber(void) { +unsigned int MKLDNNDeviceContext::GetCachedObjectsNumber(void) const { unsigned int num_entries = 0; for (auto const& l3 : *p_blobmap_) { for (auto const& l2 : *(l3.second)) { diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h index a0baf5e81122a..43c56eecad043 100644 --- a/paddle/fluid/platform/device_context.h +++ b/paddle/fluid/platform/device_context.h @@ -728,8 +728,14 @@ class MKLDNNDeviceContext : public CPUDeviceContext { using ShapeBlob = umap_key_string_t; using BlobMap = umap_value_smart_t; - using ExecMap = std::unordered_map< - void*, std::vector, KeyBlob::iterator>>>; + // Auxillary two-level structure (shape, executor) to easier control + // clearing cache objects related to specific executor + + using ExecKey = void*; + using ExecMapCacheIterPair = std::pair, KeyBlob::iterator>; + using ExecMap = + std::unordered_map>; + using ExecShape = std::unordered_map>; explicit MKLDNNDeviceContext(CPUPlace place); @@ -738,6 +744,7 @@ class MKLDNNDeviceContext : public CPUDeviceContext { // Register object to currently used executor's map void LinkEntryWithExecutor(BlobPtr_t, KeyBlob::iterator) const; + void RemoveShapeEntriesWithExecutor(void) const; // Remove all entries from the blob map void ResetBlobMap(void* ptr); @@ -752,7 +759,7 @@ class MKLDNNDeviceContext : public CPUDeviceContext { void SetBlob(const std::string& name, std::shared_ptr data) const; // Calculate number of oneDNN objects cached - unsigned int GetCachedObjectsNumber(void); + unsigned int GetCachedObjectsNumber(void) const; // Find a saved blob. Return nullptr if not found std::shared_ptr GetBlob(const std::string& name) const; @@ -765,7 +772,7 @@ class MKLDNNDeviceContext : public CPUDeviceContext { std::shared_ptr p_blobmap_; // Map key is pointer of executor and value is a data(iterator in map) needed // to erase - std::shared_ptr p_exec_items_; + std::shared_ptr p_exec_items_; std::shared_ptr p_mutex_; bool block_next_cache_clearing_ = false; }; From 172f27191002b21a31c1cbb2df092e4446b67606 Mon Sep 17 00:00:00 2001 From: lilong12 Date: Wed, 16 Jun 2021 22:01:40 +0800 Subject: [PATCH 115/156] bug fix, test=develop (#33595) --- .../paddle/distributed/fleet/meta_optimizers/sharding/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py index f4ceb2d287a56..a628105de0f4f 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py @@ -555,7 +555,7 @@ def save_persistables(exe, dirname, main_program, filename=None): """ # TODO (JZ-LIANG) revise this for uniform mixed parallelism if main_program._pipeline_opt: - main_program = main_program._pipeline_opt['section_program']['program'] + main_program = main_program._pipeline_opt['section_program'] def is_opt_vars(var): # NOTE(JZ-LIANG): The checks should be updated when add new compatible optimizer From 7be50f9051f6ea87e779e462c013ecaae348fcf4 Mon Sep 17 00:00:00 2001 From: lilong12 Date: Wed, 16 Jun 2021 22:02:15 +0800 Subject: [PATCH 116/156] update, test=develop (#33588) --- .../meta_optimizers/sharding_optimizer.py | 64 ++++++------------- 1 file changed, 19 insertions(+), 45 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py index 82e54a89e104f..d5592cf3e05ed 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py @@ -428,59 +428,33 @@ def _init_comm(self): # pp ring if self.pp_degree > 1: - if self.schedule_mode == 'F-then-B': # GPipe - self._collective_helper._init_communicator( - self._startup_program, - self.current_endpoint, - self.pp_group_endpoints, - self.pp_rank, - self.pp_ring_id, - False, - global_ring_id=self.global_ring_id, - sync=False) - # append_naive_sync(startup_block, self.startup_prog_sync_var, - # self.global_ring_id) + for pair in self.pipeline_pair: + pair_key = pair[0] * 1000 + pair[1] + ring_id = self.pp_ring_map[pair_key] + print("pp pair:{}, ring_id: {}".format(pair, ring_id)) + if self.pp_rank not in pair: continue + pp_group_endpoints = [ + self.pp_group_endpoints[pair[0]], + self.pp_group_endpoints[pair[1]], + ] + if pair[0] < pair[1]: + start_ring_id = self.pp_ring_id + pair[1] - pair[0] - 1 + else: + start_ring_id = self.pp_ring_id + 2 + pair[0] - pair[1] - 1 + pp_rank = 0 if self.pp_rank == pair[0] else 1 self._collective_helper._init_communicator( self._startup_program, self.current_endpoint, - self.pp_group_endpoints, - self.pp_rank, - self.pp_ring_id + 2, + pp_group_endpoints, + pp_rank, + ring_id, False, global_ring_id=self.global_ring_id, sync=False) # append_naive_sync(startup_block, self.startup_prog_sync_var, # self.global_ring_id) - else: - assert self.schedule_mode == '1F1B' - for pair in self.pipeline_pair: - pair_key = pair[0] * 1000 + pair[1] - ring_id = self.pp_ring_map[pair_key] - print("pp pair:{}, ring_id: {}".format(pair, ring_id)) - if self.pp_rank not in pair: continue - pp_group_endpoints = [ - self.pp_group_endpoints[pair[0]], - self.pp_group_endpoints[pair[1]], - ] - if pair[0] < pair[1]: - start_ring_id = self.pp_ring_id + pair[1] - pair[0] - 1 - else: - start_ring_id = self.pp_ring_id + 2 + pair[0] - pair[ - 1] - 1 - pp_rank = 0 if self.pp_rank == pair[0] else 1 - self._collective_helper._init_communicator( - self._startup_program, - self.current_endpoint, - pp_group_endpoints, - pp_rank, - ring_id, - False, - global_ring_id=self.global_ring_id, - sync=False) - # append_naive_sync(startup_block, self.startup_prog_sync_var, - # self.global_ring_id) - - # TODO (JZ-LIANG) to unify this shit + + # TODO (JZ-LIANG) to unify this shit assert self.pp_rank_ == self.pp_rank, "pp rank for pp opt [{}], pp rank for sharding opt [{}]".format( self.pp_rank_, self.pp_rank) From bb5963da14ce6554fcef7a8ae1949b9843fc1b8a Mon Sep 17 00:00:00 2001 From: lilong12 Date: Wed, 16 Jun 2021 22:05:05 +0800 Subject: [PATCH 117/156] [CP] add a strategy to run program with fleet (#33511) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add raw program meta optimizer (#32597) * add raw program, test=develop * add precision unitest for executor all reduce (#33339) * fix dp (#33297) Co-authored-by: Yuang Liu Co-authored-by: 李季 <2042519524@qq.com> --- .../framework/distributed_strategy.proto | 1 + .../fleet/base/distributed_strategy.py | 26 +++ .../fleet/meta_optimizers/__init__.py | 1 + .../meta_optimizers/raw_program_optimizer.py | 197 ++++++++++++++++++ .../fluid/tests/unittests/CMakeLists.txt | 12 +- .../dist_fleet_raw_program_optimizer.py | 109 ++++++++++ .../fluid/tests/unittests/test_dist_base.py | 76 ++++++- .../test_dist_fleet_raw_program_optimizer.py | 45 ++++ .../test_fleet_raw_program_meta_optimizer.py | 53 +++++ .../unittests/test_raw_program_optimizer.py | 77 +++++++ 10 files changed, 592 insertions(+), 5 deletions(-) create mode 100755 python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py create mode 100644 python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py create mode 100644 python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py create mode 100644 python/paddle/fluid/tests/unittests/test_fleet_raw_program_meta_optimizer.py create mode 100644 python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 38831192c8c2b..181e3b6885380 100644 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -175,6 +175,7 @@ message DistributedStrategy { optional float last_comm_group_size_MB = 27 [ default = 1 ]; optional bool find_unused_parameters = 28 [ default = false ]; optional bool tensor_parallel = 29 [ default = false ]; + optional bool without_graph_optimization = 30 [ default = false ]; optional RecomputeConfig recompute_configs = 101; optional AMPConfig amp_configs = 102; diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index 640bc00cb6c57..f9cd623afef76 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -827,6 +827,32 @@ def sharding_configs(self, configs): "sharding_configs") assign_configs_value(self.strategy.sharding_configs, configs) + @property + def without_graph_optimization(self): + """ + Run program using Executor other than ParallelExecutor. + + Examples: + + .. code-block:: python + + import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() + strategy.without_graph_optimization = True + + """ + return self.strategy.without_graph_optimization + + @without_graph_optimization.setter + @is_strict_auto + def without_graph_optimization(self, flag): + if isinstance(flag, bool): + self.strategy.without_graph_optimization = flag + else: + print( + "WARNING: without_graph_optimization should have value of bool type" + ) + @property def pipeline(self): """ diff --git a/python/paddle/distributed/fleet/meta_optimizers/__init__.py b/python/paddle/distributed/fleet/meta_optimizers/__init__.py index 827835fde20e3..1788e044fe885 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/__init__.py +++ b/python/paddle/distributed/fleet/meta_optimizers/__init__.py @@ -28,3 +28,4 @@ from .dygraph_optimizer import HybridParallelOptimizer from .dygraph_optimizer import HybridParallelGradScaler from .tensor_parallel_optimizer import TensorParallelOptimizer +from .raw_program_optimizer import RawProgramOptimizer diff --git a/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py new file mode 100755 index 0000000000000..b232d8c9c49fc --- /dev/null +++ b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py @@ -0,0 +1,197 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +from __future__ import print_function +from __future__ import division +import os + +import paddle.fluid as fluid +from paddle.fluid import core, unique_name +from ..base.private_helper_function import wait_server_ready +from .meta_optimizer_base import MetaOptimizerBase +from .common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY, CollectiveHelper, is_loss_grad_op, is_backward_op, is_optimizer_op + + +class RawProgramOptimizer(MetaOptimizerBase): + def __init__(self, optimizer): + super(RawProgramOptimizer, self).__init__(optimizer) + self.inner_opt = optimizer + self.meta_optimizers_white_list = [ + "RecomputeOptimizer", + "AMPOptimizer", + ] + self.meta_optimizers_black_list = ["GraphExecutionOptimizer", ] + self.global_ring_id = 0 + + def _set_basic_info(self, loss, role_maker, user_defined_optimizer, + user_defined_strategy): + super(RawProgramOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy) + self.without_graph_optimization = user_defined_strategy.without_graph_optimization + + def _can_apply(self): + if not self.role_maker._is_collective: + return False + + if self.without_graph_optimization == True: + return True + return False + + def _disable_strategy(self, dist_strategy): + dist_strategy.without_graph_optimization = False + + def _enable_strategy(self, dist_strategy, context): + dist_strategy.without_graph_optimization = True + + def _broadcast_params(self, ring_id): + block = self.startup_program.global_block() + param = None + for param in block.iter_parameters(): + if param.is_distributed: + continue + + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward + }) + + if not param: return # no parameter on this device + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward}) + + def _get_process_group_info(self): + # global ring info + self.global_endpoints = self.endpoints + self.global_rank = self.rank + self.global_nranks = self.nranks + + def _init_process_group(self): + self._get_process_group_info() + collective_helper = CollectiveHelper(self.role_maker, wait_port=False) + # Create global ring for all gpus (ring_id = 0) + collective_helper._init_communicator( + self.startup_program, self.current_endpoint, self.global_endpoints, + self.global_rank, self.global_ring_id, True, self.global_ring_id, + True) + self._broadcast_params(self.global_ring_id) + + def minimize_impl(self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None): + self.endpoints = self.role_maker._get_trainer_endpoints() + self.current_endpoint = self.endpoints[self.role_maker._worker_index()] + self.rank = self.role_maker._worker_index() + self.nranks = self.role_maker._worker_num() + if startup_program is None: + startup_program = fluid.default_startup_program() + self.startup_program = startup_program + + block = loss.block + program = block.program + self.main_program = program + + optimize_ops, params_grads = self.inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set) + if self.nranks == 1: + return optimize_ops, params_grads + self._init_process_group() + + self.main_program = program + if self.nranks > 1: + self._transpile_main_program(loss) + return optimize_ops, params_grads + + def _transpile_main_program(self, loss): + self._insert_loss_grad_ops(loss) + self._insert_allreduce_ops() + + def _insert_loss_grad_ops(self, loss): + """ + In order to keep the learning rate consistent in different numbers of + training workers, we scale the loss grad by the number of workers + """ + block = self.main_program.global_block() + for idx, op in reversed(list(enumerate(block.ops))): + if is_loss_grad_op(op): + loss_grad_var = block.vars[op.output_arg_names[0]] + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / self.nranks, + OP_ROLE_KEY: OpRole.Backward + }) + + def _insert_allreduce_ops(self): + block = self.main_program.global_block() + ring_id = self.global_ring_id + grad = None + for idx, op in reversed(list(enumerate(block.ops))): + if is_backward_op(op) and \ + OP_ROLE_VAR_KEY in op.attr_names: + op_role_var = op.attr(OP_ROLE_VAR_KEY) + if len(op_role_var) == 0: + continue + assert len(op_role_var) % 2 == 0 + offset = 1 + for i in range(0, len(op_role_var), 2): + param_name = op_role_var[i] + param = block.var(param_name) + grad_name = op_role_var[i + 1] + grad = block.var(grad_name) + if param.is_distributed: + continue + + block._insert_op( + idx + offset, + type='c_sync_calc_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={OP_ROLE_KEY: OpRole.Backward, }) + offset += 1 + block._insert_op( + idx + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward + }) + + if grad is None: + return + + for idx, op in enumerate(block.ops): + if is_optimizer_op(op): + block._insert_op( + idx, + type='c_sync_comm_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward}) + break diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 37bcac4957493..8341e9b93e67c 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -17,6 +17,7 @@ list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding) list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding_over_height) list(APPEND DIST_TEST_OPS test_parallel_dygraph_transformer) list(APPEND DIST_TEST_OPS test_fleet_pipeline_meta_optimizer) +list(APPEND DIST_TEST_OPS test_fleet_raw_program_meta_optimizer) list(APPEND DIST_TEST_OPS test_fleet_graph_execution_meta_optimizer) list(APPEND DIST_TEST_OPS test_gen_nccl_id_op) list(APPEND DIST_TEST_OPS test_parallel_dygraph_unused_variables) @@ -54,6 +55,7 @@ list(APPEND MIXED_DIST_TEST_OPS test_fleet_base_2) list(APPEND MIXED_DIST_TEST_OPS test_fleet_base_3) list(APPEND MIXED_DIST_TEST_OPS test_fleet_recompute_meta_optimizer) list(APPEND MIXED_DIST_TEST_OPS test_fleet_pipeline_meta_optimizer) +list(APPEND MIXED_DIST_TEST_OPS test_fleet_raw_program_meta_optimizer) list(APPEND MIXED_DIST_TEST_OPS test_fleet_amp_meta_optimizer) list(APPEND MIXED_DIST_TEST_OPS test_fleet_amp_init) list(APPEND MIXED_DIST_TEST_OPS test_fleet_gradient_merge_meta_optimizer) @@ -100,6 +102,7 @@ if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32) LIST(REMOVE_ITEM TEST_OPS test_collective_sendrecv_api) LIST(REMOVE_ITEM TEST_OPS test_collective_wait) LIST(REMOVE_ITEM TEST_OPS test_memcpy_op) + LIST(REMOVE_ITEM TEST_OPS test_raw_program_optimizer) endif() if(WIN32) @@ -571,7 +574,7 @@ endif() py_test_modules(test_parallel_executor_crf MODULES test_parallel_executor_crf) # Coverage pipeline use cuda 10.1 now, profiler will random hang in cuda 10.1, # see https://github.com/PaddlePaddle/Paddle/issues/29082 for details. -# We guess there are some bugs in cuda 10.1 or 10.2, +# We guess there are some bugs in cuda 10.1 or 10.2, # since this unittest is stable in cuda 11 (py3 pipeline) now. if(NOT WITH_COVERAGE) py_test_modules(test_parallel_executor_profiler MODULES test_parallel_executor_profiler) @@ -596,8 +599,8 @@ py_test_modules(test_fuse_bn_act_pass MODULES test_fuse_bn_act_pass ENVS FLAGS_c py_test_modules(test_fuse_bn_add_act_pass MODULES test_fuse_bn_add_act_pass ENVS FLAGS_cudnn_deterministic=1 FLAGS_cudnn_batchnorm_spatial_persistent=1 FLAGS_conv_workspace_size_limit=1000) # NOTE: These unittests will appear NaN steadily in windows CI. After analysis, -# it is found that windows CI will run all the training unittests with the ON_INFER option turned on, -# which will not appear in other CIs. The calculation behavior of some ops in inference mode is +# it is found that windows CI will run all the training unittests with the ON_INFER option turned on, +# which will not appear in other CIs. The calculation behavior of some ops in inference mode is # inconsistent with that in non-inference mode. if(NOT ON_INFER) py_test_modules(test_parallel_executor_seresnext_base_cpu MODULES test_parallel_executor_seresnext_base_cpu) @@ -640,7 +643,7 @@ if (WITH_XPU) add_subdirectory(xpu) endif() -# dist xpu tests: +# dist xpu tests: if (WITH_XPU_BKCL) py_test(test_collective_reduce_api_xpu SRCS "test_collective_reduce_api.py") py_test(test_collective_allreduce_api_xpu SRCS "test_collective_allreduce_api.py") @@ -708,6 +711,7 @@ if (WITH_DISTRIBUTE) set_tests_properties(test_dist_fleet_ctr2 PROPERTIES TIMEOUT 200) set_tests_properties(test_dist_fleet_sparse_embedding_ctr PROPERTIES TIMEOUT 200) set_tests_properties(test_dist_fleet_infer PROPERTIES TIMEOUT 200) + set_tests_properties(test_dist_fleet_raw_program_optimizer PROPERTIES TIMEOUT 120) endif() if (WITH_DISTRIBUTE AND NOT APPLE) diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py new file mode 100644 index 0000000000000..575c07390a35b --- /dev/null +++ b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py @@ -0,0 +1,109 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from test_dist_base import TestDistRunnerBase, runtime_main +import unittest +import paddle +import os +import paddle.distributed.fleet as fleet +import paddle.distributed.fleet.base.role_maker as role_maker +import numpy as np +from functools import reduce +import paddle.fluid as fluid + +paddle.enable_static() + +DTYPE = "float32" +paddle.dataset.mnist.fetch() + +# Fix seed for test +fluid.default_startup_program().random_seed = 1 +fluid.default_main_program().random_seed = 1 + + +def cnn_model(data): + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=data, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( + value=0.01))) + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( + value=0.01))) + + SIZE = 10 + input_shape = conv_pool_2.shape + param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] + scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + + predict = fluid.layers.fc( + input=conv_pool_2, + size=SIZE, + act="softmax", + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01))) + return predict + + +class TestFleetMetaOptimizerPrecision(TestDistRunnerBase): + def get_model(self, batch_size=2, single_device=False): + # Input data + images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + + # Train program + predict = cnn_model(images) + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(x=cost) + + # Evaluator + batch_size_tensor = fluid.layers.create_tensor(dtype='int64') + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor) + + test_program = fluid.default_main_program().clone(for_test=True) + + # Reader + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size) + + optimizer = paddle.fluid.optimizer.Adam(0.01) + if single_device: + optimizer.minimize(avg_cost) + else: + role = role_maker.PaddleCloudRoleMaker(is_collective=True) + fleet.init(role) + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.without_graph_optimization = True + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + return test_program, avg_cost, train_reader, test_reader, batch_acc, predict + + +if __name__ == "__main__": + runtime_main(TestFleetMetaOptimizerPrecision) diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index edc510e4e766d..78b06bd5333d7 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -186,6 +186,76 @@ def run_pipeline_trainer(self, args): fleet.save_inference_model(exe, infer_save_dir_fleet, feeded_var_names, [avg_cost]) + def run_use_fleet_api_20_trainer(self, args): + """ + 1. remove codes for DistributedStrategy and leave the DistributedStrategy part to get_model() + 2. to run with fleet 2.0 api, set flags _use_fleet_api and _use_fleet_api_20 to True + 3. for now, not support test for model save + """ + assert args.update_method == "nccl2" or "bkcl" + + self.lr = args.lr + print_to_err("use_fleet 2.0", "fleet.node_num:") + + test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ + self.get_model(batch_size=args.batch_size) + + if fluid.core.is_compiled_with_cuda(): + device_id = int(os.getenv("FLAGS_selected_gpus", "0")) + place = fluid.CUDAPlace(device_id) + elif fluid.core.is_compiled_with_xpu(): + device_id = int(os.getenv("FLAGS_selected_xpus", "0")) + place = fluid.XPUPlace(device_id) + else: + raise ValueError( + "fleet dygraph api must in paddlepaddle-xpu or paddlepaddle-gpu." + ) + + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) + eprint(type(self).__name__, "run worker startup program done.") + + feed_var_list = [ + var + for var in fluid.default_main_program().global_block().vars.values() + if var.is_data + ] + + eprint("feed_var_list:", feed_var_list) + + if feed_var_list[0].name == 'label': + feed_var_list = feed_var_list[::-1] + + feeder = fluid.DataFeeder(feed_var_list, place) + reader_generator = train_reader() + + def get_data(): + origin_batch = next(reader_generator) + if args.update_method != "local" and args.use_reader_alloc: + new_batch = [] + for offset, item in enumerate(origin_batch): + if offset % 2 == args.trainer_id: + new_batch.append(item) + return new_batch + else: + return origin_batch + + print_to_err(type(self).__name__, "begin to train on trainer") + out_losses = [] + for i in six.moves.xrange(RUN_STEP): + loss, = exe.run(fluid.default_main_program(), + fetch_list=[avg_cost.name], + feed=feeder.feed(get_data())) + out_losses.append(loss[0]) + print_to_err(type(self).__name__, "run step %d finished" % i) + print_to_err(type(self).__name__, "trainer run finished") + print_to_err(type(self).__name__, "dist losses: {}".format(out_losses)) + + if six.PY2: + print(pickle.dumps(out_losses)) + else: + sys.stdout.buffer.write(pickle.dumps(out_losses)) + def run_use_fleet_api_trainer(self, args): assert args.update_method == "nccl2" or "bkcl" @@ -630,6 +700,7 @@ def runtime_main(test_class): parser.add_argument('--use_hallreduce', action='store_true') parser.add_argument('--use_pipeline', action='store_true') parser.add_argument('--use_fleet_api', action='store_true') + parser.add_argument('--use_fleet_api_20', action='store_true') parser.add_argument('--use_local_sgd', action='store_true') parser.add_argument('--ut4grad_allreduce', action='store_true') parser.add_argument( @@ -671,6 +742,8 @@ def runtime_main(test_class): model.run_pserver(args) elif args.use_fleet_api: model.run_use_fleet_api_trainer(args) + elif args.use_fleet_api_20: + model.run_use_fleet_api_20_trainer(args) elif args.use_pipeline: model.run_pipeline_trainer(args) else: @@ -734,6 +807,7 @@ def setUp(self): self._nccl_comm_num = 1 self._enable_backward_deps = False self._use_fleet_api = False + self._use_fleet_api_20 = False self._use_local_sgd = False self._ut4grad_allreduce = False self._use_hallreduce = False @@ -1060,7 +1134,7 @@ def _get_nccl2_trainer_cmd(self, model, ep, update_method, trainer_id, tr_cmd += " --fuse_all_reduce {}".format(self._fuse_all_reduce) if self._use_fleet_api: - tr_cmd += " --use_fleet_api" + tr_cmd += " --use_fleet_api_20" if self._use_fleet_api_20 else " --use_fleet_api" if self._use_local_sgd: tr_cmd += " --use_local_sgd" if self._ut4grad_allreduce: diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py new file mode 100644 index 0000000000000..e729bfe053752 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py @@ -0,0 +1,45 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from test_dist_base import TestDistBase +import paddle +import os + +paddle.enable_static() +flag_name = os.path.splitext(__file__)[0] + + +class TestFleetMetaOptimizerPrecision(TestDistBase): + def _setup_config(self): + self._sync_mode = True + self._use_reduce = False + self._use_reader_alloc = False + self._nccl2_mode = True + self._nccl2_reduce_layer = True + self._use_fleet_api = True + self._use_fleet_api_20 = True + + def test_dist_train(self): + import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): + self.check_with_place( + "dist_fleet_raw_program_optimizer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_raw_program_meta_optimizer.py b/python/paddle/fluid/tests/unittests/test_fleet_raw_program_meta_optimizer.py new file mode 100644 index 0000000000000..604109b262d6c --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_fleet_raw_program_meta_optimizer.py @@ -0,0 +1,53 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import paddle +import os + +paddle.enable_static() + + +class TestFleetMetaOptimizer(unittest.TestCase): + def setUp(self): + os.environ["PADDLE_TRAINER_ID"] = "1" + os.environ[ + "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + + def test_pipeline_optimizer(self): + import paddle.distributed.fleet as fleet + import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) + fleet.init(role) + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32') + input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') + fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') + + fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') + prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.fluid.layers.mean(x=cost) + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.without_graph_optimization = True + + optimizer = paddle.fluid.optimizer.Adam(0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py new file mode 100644 index 0000000000000..34930e3577b9b --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py @@ -0,0 +1,77 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest + +import paddle +import paddle.fluid as fluid +import paddle.fluid.core as core +import paddle.distributed.fleet as fleet +import numpy as np +import os + + +class TestRawProgramOptimizer(unittest.TestCase): + def setUp(self): + os.environ["PADDLE_TRAINER_ID"] = "0" + os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" + + def mlp(self, input_x, input_y, hid_dim=128, label_dim=2): + fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh') + fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh') + prediction = paddle.static.nn.fc(x=[fc_2], + size=label_dim, + activation='softmax') + cost = paddle.nn.functional.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.mean(x=cost) + return avg_cost + + def gen_data(self): + return { + "x": np.random.random(size=(128, 32)).astype('float32'), + "y": np.random.randint( + 2, size=(128, 1)).astype('int64') + } + + def test_single_gpu(self): + paddle.enable_static() + fleet.init(is_collective=True) + sharding_program = paddle.static.Program() + sharding_startup_program = paddle.static.Program() + strategy = fleet.DistributedStrategy() + strategy.without_graph_optimization = True + with fluid.program_guard(sharding_program, sharding_startup_program): + with fluid.unique_name.guard(): + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32') + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64') + cost = self.mlp(input_x=input_x, input_y=input_y) + output_name = cost.name + optimizer = fleet.distributed_optimizer(fluid.optimizer.Adam(), + strategy) + optimizer.minimize(cost) + + trainer_id = fleet.worker_index() + exe = paddle.static.Executor(paddle.CUDAPlace(trainer_id)) + rank = fleet.worker_index() + exe.run(sharding_startup_program) + exe.run(program=sharding_program, feed=self.gen_data()) + + +if __name__ == "__main__": + unittest.main() From 63aeb02dfc50145a0911920f1cb97978abf4e121 Mon Sep 17 00:00:00 2001 From: TTerror Date: Wed, 16 Jun 2021 22:09:25 +0800 Subject: [PATCH 118/156] fix gather op and add logsumexp op on kunlun (#32931) (#33592) * fix gather op and add logsumexp op on kunlun * update xpu depence * update tests and fix elementwise_add --- cmake/external/xpu.cmake | 2 +- .../elementwise/elementwise_add_op_xpu.cc | 7 +- paddle/fluid/operators/gather_op_xpu.cc | 84 ++++++++-------- .../operators/reduce_ops/logsumexp_op_xpu.cc | 74 ++++++++++++++ .../tests/unittests/xpu/test_gather_op_xpu.py | 57 +++++------ .../unittests/xpu/test_logsumexp_op_xpu.py | 97 +++++++++++++++++++ 6 files changed, 238 insertions(+), 83 deletions(-) create mode 100644 paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc create mode 100644 python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index f846623602ed7..a03ff7d22dcad 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -13,7 +13,7 @@ if(NOT XPU_SDK_ROOT) elseif(WITH_SUNWAY) SET(XPU_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/sunway/xpu_2021_01_13.tar.gz" CACHE STRING "" FORCE) else() - SET(XPU_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/xpu_2021_04_09.tar.gz" CACHE STRING "" FORCE) + SET(XPU_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/xpu_2021_05_19.tar.gz" CACHE STRING "" FORCE) endif() SET(XPU_SOURCE_DIR "${THIRD_PARTY_PATH}/xpu") diff --git a/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc b/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc index 8d99aa2798568..8b902acebb4c5 100644 --- a/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc +++ b/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc @@ -141,6 +141,7 @@ class ElementwiseAddGradXPUKernel : public ElemwiseGradKernel { } } + const T* dz_data = dz->data(); T* dx_data = nullptr; T* dy_data = nullptr; if (dx) { @@ -152,9 +153,9 @@ class ElementwiseAddGradXPUKernel : public ElemwiseGradKernel { auto& dev_ctx = ctx.template device_context(); - int ret = xpu::broadcast_add_grad(dev_ctx.x_context(), dx_data, dx_data, - dx_data, dz->data(), dy_data, - dx_data, x_dims_vec, y_dims_vec); + int ret = xpu::broadcast_add_grad(dev_ctx.x_context(), dz_data, dz_data, + dz_data, dz_data, dy_data, dx_data, + x_dims_vec, y_dims_vec); PADDLE_ENFORCE_EQ( ret, xpu::SUCCESS, platform::errors::External( diff --git a/paddle/fluid/operators/gather_op_xpu.cc b/paddle/fluid/operators/gather_op_xpu.cc index ae3d0f2633bb1..6d1dac8304050 100644 --- a/paddle/fluid/operators/gather_op_xpu.cc +++ b/paddle/fluid/operators/gather_op_xpu.cc @@ -40,16 +40,6 @@ class GatherOpXPUKernel : public framework::OpKernel { output->mutable_data(ctx.GetPlace()); if (x->numel() == 0) return; - // check index type is INT32 - const auto &index_type = index->type(); - bool index_type_match = index_type == framework::proto::VarType::INT32; - PADDLE_ENFORCE_EQ( - index_type_match, true, - platform::errors::InvalidArgument( - "XPU only support INT32, it holds %s, but desires to be %s", - paddle::framework::DataTypeToString(index_type), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT32))); const auto index_dims = index->dims(); if (index_dims.size() == 2) { @@ -65,14 +55,26 @@ class GatherOpXPUKernel : public framework::OpKernel { "The index should be 1D, when it is not 2D, but we get %d", index_dims.size())); } - int slice_size = x->numel() / x->dims()[0]; + std::vector xshape(x->dims().size()); + for (int i = 0; i < x->dims().size(); ++i) { + xshape[i] = x->dims()[i]; + } + auto &dev_ctx = ctx.template device_context(); - int r = - xpu::gather(dev_ctx.x_context(), x->data(), index->data(), - index->dims()[0], slice_size, output->data()); - PADDLE_ENFORCE_EQ( - r, xpu::Error_t::SUCCESS, - platform::errors::External("XPU kernel error! error code=%d", r)); + int r = XPU_SUCCESS; + if (index->type() == framework::proto::VarType::INT32) { + r = xpu::gather(dev_ctx.x_context(), x->data(), + index->data(), output->data(), xshape, + index->dims()[0], 0); + } else { + r = xpu::gather(dev_ctx.x_context(), x->data(), + index->data(), output->data(), + xshape, index->dims()[0], 0); + } + PADDLE_ENFORCE_EQ(r, xpu::Error_t::SUCCESS, + platform::errors::External( + "XPU gather kernel return wrong value[%d %s]", r, + XPUAPIErrorMsg[r])); } }; @@ -93,30 +95,11 @@ class GatherGradOpXPUKernel : public framework::OpKernel { PADDLE_THROW(platform::errors::InvalidArgument( "Now, it doesn't support XPU with Axis.")); } - - dx->mutable_data(ctx.GetPlace()); - const int zero = 0; - int r_dx = xpu::memset(dev_ctx.x_context(), dx->data(), zero, - dx->numel() * sizeof(T)); - PADDLE_ENFORCE_EQ( - r_dx, xpu::Error_t::SUCCESS, - platform::errors::External("XPU kernel error! error code=%d", r_dx)); - if (dout->numel() == 0) { return; } - bool overwrite = ctx.Attr("overwrite"); - // check index type is INT32 - const auto &index_type = index->type(); - bool index_type_match = index_type == framework::proto::VarType::INT32; - PADDLE_ENFORCE_EQ( - index_type_match, true, - platform::errors::InvalidArgument( - "XPU only support INT32, it holds %s, but desires to be %s", - paddle::framework::DataTypeToString(index_type), - paddle::framework::DataTypeToString( - framework::proto::VarType::INT32))); + bool overwrite = ctx.Attr("overwrite"); const auto index_dims = index->dims(); if (index_dims.size() == 2) { PADDLE_ENFORCE_EQ( @@ -131,16 +114,27 @@ class GatherGradOpXPUKernel : public framework::OpKernel { "The index should be 1D, when it is not 2D, but we get %d", index_dims.size())); } + std::vector xshape(dx->dims().size()); + for (int i = 0; i < dx->dims().size(); ++i) { + xshape[i] = dx->dims()[i]; + } - int index_size = index_dims[0]; - int slice_size = dout->numel() / dout->dims()[0]; + dx->mutable_data(ctx.GetPlace()); - int r = xpu::scatter(dev_ctx.x_context(), dout->data(), - index->data(), index_size, slice_size, - dx->data(), overwrite); - PADDLE_ENFORCE_EQ( - r, xpu::Error_t::SUCCESS, - platform::errors::External("XPU kernel error! error code=%d", r)); + int r = XPU_SUCCESS; + if (index->type() == framework::proto::VarType::INT32) { + r = xpu::gather_grad(dev_ctx.x_context(), dout->data(), + index->data(), dx->data(), xshape, + index->dims()[0], 0, overwrite); + } else { + r = xpu::gather_grad(dev_ctx.x_context(), dout->data(), + index->data(), dx->data(), + xshape, index->dims()[0], 0, overwrite); + } + PADDLE_ENFORCE_EQ(r, xpu::Error_t::SUCCESS, + platform::errors::External( + "XPU gather grad kernel return wrong value[%d %s]", r, + XPUAPIErrorMsg[r])); } }; diff --git a/paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc b/paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc new file mode 100644 index 0000000000000..9cc8ac200b8ee --- /dev/null +++ b/paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc @@ -0,0 +1,74 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef PADDLE_WITH_XPU + +#include "paddle/fluid/operators/reduce_ops/logsumexp_op.h" +#include "paddle/fluid/platform/device_context.h" +#include "paddle/fluid/platform/xpu_header.h" + +namespace paddle { +namespace operators { + +template +class XPULogsumexpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* input = context.Input("X"); + auto* output = context.Output("Out"); + + auto axis = context.Attr>("axis"); + auto reduce_all = context.Attr("reduce_all"); + + const auto& input_dim_size = input->dims().size(); + // The dims has full dim, set the reduce_all is True + reduce_all |= (static_cast(axis.size()) == input_dim_size); + + const T* input_data = input->data(); + T* output_data = output->mutable_data(context.GetPlace()); + + std::vector axis_shape; + std::vector xdims(input_dim_size); + for (int i = 0; i < input_dim_size; ++i) { + xdims[i] = input->dims()[i]; + } + if (reduce_all) { + for (int i = 0; i < input_dim_size; ++i) { + axis_shape.push_back(i); + } + } else { + for (size_t i = 0; i < axis.size(); ++i) { + int rdim = axis[i] < 0 ? axis[i] + input_dim_size : axis[i]; + axis_shape.push_back(rdim); + } + } + + auto& dev_ctx = context.template device_context(); + int r = xpu::logsumexp(dev_ctx.x_context(), input_data, output_data, + xdims, axis_shape); + PADDLE_ENFORCE_EQ(r, xpu::Error_t::SUCCESS, + platform::errors::External( + "XPU logsumexp kernel error! error value[%d %]", r, + XPUAPIErrorMsg[r])); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_XPU_KERNEL( + logsumexp, + ops::XPULogsumexpKernel); +#endif diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py index 9bea33e484e19..d33cb2157b03b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py @@ -13,13 +13,18 @@ # limitations under the License. from __future__ import print_function +import unittest import sys sys.path.append("..") -import unittest + import numpy as np -from op_test import OpTest + import paddle import paddle.fluid as fluid +from op_test import OpTest +from op_test_xpu import XPUOpTest + +paddle.enable_static() def gather_numpy(x, index, axis): @@ -29,37 +34,12 @@ def gather_numpy(x, index, axis): return gather -class TestGatherOp(OpTest): - def setUp(self): - self.op_type = "gather" - self.config() - xnp = np.random.random(self.x_shape).astype(self.x_type) - self.inputs = { - 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) - } - self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(['X'], 'Out') - - def config(self): - """ - For multi-dimension input - """ - self.x_shape = (10, 20) - self.x_type = "float64" - self.index = [1, 3, 5] - self.index_type = "int32" - - -class TestXPUGatherOp(OpTest): +class TestXPUGatherOp(XPUOpTest): def setUp(self): + self.dtype = "float32" self.op_type = "gather" - self.dtype = np.float32 + self.use_xpu = True + self.use_mkldnn = False self.attrs = {'use_xpu': True} self.config() @@ -71,12 +51,12 @@ def setUp(self): self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} def test_check_output(self): - if self.dtype == np.float32 and paddle.is_compiled_with_xpu(): + if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_output_with_place(place) def test_check_grad(self): - if self.dtype == np.float32 and paddle.is_compiled_with_xpu(): + if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place(place, ['X'], 'Out') @@ -85,7 +65,7 @@ def config(self): For multi-dimension input """ self.x_shape = (10, 20) - self.x_type = self.dtype + self.x_type = "float32" self.index = [1, 3, 5] self.index_type = "int32" @@ -150,5 +130,14 @@ def config(self): self.index_type = "int32" +class TestCase7(TestXPUGatherOp): + def config(self): + self.x_shape = (10, 20) + self.attrs = {'use_xpu': True, 'overwrite': True} + self.x_type = "float32" + self.index = [1, 3] + self.index_type = "int64" + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py new file mode 100644 index 0000000000000..c4e1363bd9c94 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py @@ -0,0 +1,97 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import unittest +import sys +sys.path.append("..") +import numpy as np +from op_test import OpTest +from op_test_xpu import XPUOpTest + +paddle.enable_static() + + +def ref_logsumexp(x, axis=None, keepdim=False, reduce_all=False): + if isinstance(axis, int): + axis = (axis, ) + elif isinstance(axis, list): + axis = tuple(axis) + if reduce_all: + axis = None + out = np.log(np.exp(x).sum(axis=axis, keepdims=keepdim)) + return out + + +class XPUTestLogsumexp(XPUOpTest): + def setUp(self): + self.op_type = 'logsumexp' + self.shape = [2, 3, 4, 5] + self.dtype = 'float32' + self.axis = [-1] + self.keepdim = False + self.reduce_all = False + self.set_attrs() + + np.random.seed(10) + x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) + out = ref_logsumexp(x, self.axis, self.keepdim, self.reduce_all) + + self.inputs = {'X': x} + self.outputs = {'Out': out} + self.attrs = { + 'axis': self.axis, + 'keepdim': self.keepdim, + 'reduce_all': self.reduce_all + } + + def set_attrs(self): + pass + + def test_check_output(self): + if paddle.is_compiled_with_xpu(): + place = paddle.XPUPlace(0) + self.check_output_with_place(place) + + def test_check_grad(self): + pass + + +class TestLogsumexp_shape(XPUTestLogsumexp): + def set_attrs(self): + self.shape = [4, 5, 6] + + +class TestLogsumexp_axis(XPUTestLogsumexp): + def set_attrs(self): + self.axis = [0, -1] + + +class TestLogsumexp_axis_all(XPUTestLogsumexp): + def set_attrs(self): + self.axis = [0, 1, 2, 3] + + +class TestLogsumexp_keepdim(XPUTestLogsumexp): + def set_attrs(self): + self.keepdim = True + + +class TestLogsumexp_reduce_all(XPUTestLogsumexp): + def set_attrs(self): + self.reduce_all = True + + +if __name__ == '__main__': + unittest.main() From 7bbeeb59aac054aa0cfd10443a2c3ed726260b71 Mon Sep 17 00:00:00 2001 From: wangguanzhong Date: Thu, 17 Jun 2021 08:26:46 +0800 Subject: [PATCH 119/156] cherry-pick fix output padding conv (#33587) * cherry-pick fix_output_padding_conv * add repr unittest for conv --- .../tests/unittests/test_conv2d_transpose_op.py | 13 +++++++++++++ python/paddle/nn/layer/conv.py | 12 ++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py index 4e582d74c24a2..b106f7aa9c1c8 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py @@ -18,6 +18,7 @@ import numpy as np import paddle +import paddle.nn as nn paddle.enable_static() import paddle.fluid.core as core import paddle.fluid as fluid @@ -898,5 +899,17 @@ def attr_padding_with_data_format(): self.assertRaises(ValueError, attr_padding_with_data_format) +class TestConv2DTransposeRepr(unittest.TestCase): + def test_case(self): + paddle.disable_static() + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + conv = nn.Conv2DTranspose(4, 6, (3, 3), output_padding=1, stride=2) + print(conv) + y_var = conv(x_var) + y_np = y_var.numpy() + self.assertIsNotNone(y_np) + paddle.enable_static() + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/nn/layer/conv.py b/python/paddle/nn/layer/conv.py index 51eab0d1838c9..12700035e5e81 100644 --- a/python/paddle/nn/layer/conv.py +++ b/python/paddle/nn/layer/conv.py @@ -98,7 +98,7 @@ def __init__(self, 'kernel_size') self._padding = padding self._padding_mode = padding_mode - self._output_padding = output_padding + self.output_padding = output_padding if dims != 1: self._updated_padding, self._padding_algorithm = _update_padding_nd( padding, channel_last, dims) @@ -163,8 +163,8 @@ def extra_repr(self): main_str += ', padding={_padding}' if self._padding_mode is not 'zeros': main_str += ', padding_mode={_padding_mode}' - if self._output_padding != 0: - main_str += ', output_padding={_output_padding}' + if self.output_padding != 0: + main_str += ', output_padding={output_padding}' if self._dilation != [1] * len(self._dilation): main_str += ', dilation={_dilation}' if self._groups != 1: @@ -502,7 +502,7 @@ def forward(self, x, output_size=None): self.weight, bias=self.bias, output_size=output_size, - output_padding=self._output_padding, + output_padding=self.output_padding, padding=self._padding, stride=self._stride, dilation=self._dilation, @@ -810,7 +810,7 @@ def __init__(self, def forward(self, x, output_size=None): if output_size is None: - output_padding = self._output_padding + output_padding = self.output_padding else: output_padding = 0 @@ -1139,7 +1139,7 @@ def __init__(self, def forward(self, x, output_size=None): if output_size is None: - output_padding = self._output_padding + output_padding = self.output_padding else: output_padding = 0 From c3807f9e722199bcefc1b0a5103282e285720a30 Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Thu, 17 Jun 2021 10:46:56 +0800 Subject: [PATCH 120/156] fix Windows CI unstable (#33606) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复Windows CI的一些突发新增的不稳定现象： 清理服务器后台未正常退出的进程 支持外部手动修改cache缓存目录 --- paddle/scripts/paddle_build.bat | 56 ++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 5f157e28da6ef..f6c947eee0d5e 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -18,19 +18,18 @@ rem Paddle CI Task On Windows Platform rem ================================================= @ECHO ON -setlocal +setlocal enabledelayedexpansion rem -------clean up environment----------- set work_dir=%cd% -set cache_dir=%work_dir:Paddle=cache% +if not defined cache_dir set cache_dir=%work_dir:Paddle=cache% if not exist %cache_dir%\tools ( git clone https://github.com/zhouwei25/tools.git %cache_dir%\tools ) -taskkill /f /im op_function_generator.exe 2>NUL taskkill /f /im cmake.exe 2>NUL taskkill /f /im MSBuild.exe 2>NUL -taskkill /f /im CL.exe 2>NUL -taskkill /f /im Lib.exe 2>NUL +taskkill /f /im cl.exe 2>NUL +taskkill /f /im lib.exe 2>NUL taskkill /f /im link.exe 2>NUL taskkill /f /im vctip.exe 2>NUL taskkill /f /im cvtres.exe 2>NUL @@ -41,14 +40,12 @@ taskkill /f /im python.exe 2>NUL taskkill /f /im nvcc.exe 2>NUL taskkill /f /im cicc.exe 2>NUL taskkill /f /im ptxas.exe 2>NUL -taskkill /f /im test_api_impl.exe 2>NUL taskkill /f /im op_function_generator.exe 2>NUL wmic process where name="op_function_generator.exe" call terminate 2>NUL -wmic process where name="test_api_impl.exe" call terminate 2>NUL wmic process where name="cvtres.exe" call terminate 2>NUL wmic process where name="rc.exe" call terminate 2>NUL -wmic process where name="CL.exe" call terminate 2>NUL -wmic process where name="Lib.exe" call terminate 2>NUL +wmic process where name="cl.exe" call terminate 2>NUL +wmic process where name="lib.exe" call terminate 2>NUL wmic process where name="python.exe" call terminate 2>NUL rem ------initialize common variable------ @@ -66,7 +63,7 @@ if not defined WITH_PYTHON set WITH_PYTHON=ON if not defined ON_INFER set ON_INFER=ON if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=ON if not defined WITH_STATIC_LIB set WITH_STATIC_LIB=ON -if not defined WITH_TPCACHE set WITH_TPCACHE=ON +if not defined WITH_TPCACHE set WITH_TPCACHE=OFF if not defined WITH_CLCACHE set WITH_CLCACHE=OFF if not defined WITH_CACHE set WITH_CACHE=OFF if not defined WITH_UNITY_BUILD set WITH_UNITY_BUILD=OFF @@ -79,6 +76,7 @@ if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37 rem -------set cache build directory----------- rmdir build\python /s/q +rem rmdir build\paddle\fluid\pybind /s/q rmdir build\paddle_install_dir /s/q rmdir build\paddle_inference_install_dir /s/q rmdir build\paddle_inference_c_install_dir /s/q @@ -137,10 +135,11 @@ goto :CASE_%1 echo "Usage: paddle_build.bat [OPTION]" echo "OPTION:" -echo "wincheck_mkl: run Windows MKL/GPU/UnitTest CI tasks on Windows" -echo "wincheck_openbals: run Windows OPENBLAS/CPU CI tasks on Windows" +echo "wincheck_mkl: run Windows MKL/GPU PR CI tasks on Windows" +echo "wincheck_openbals: run Windows OPENBLAS/CPU PR CI tasks on Windows" echo "build_avx_whl: build Windows avx whl package on Windows" echo "build_no_avx_whl: build Windows no avx whl package on Windows" +echo "build_inference_lib: build Windows inference library on Windows" exit /b 1 rem ------PR CI windows check for MKL/GPU---------- @@ -200,6 +199,7 @@ goto:success rem ------Build windows inference library------ :CASE_build_inference_lib +set ON_INFER=ON set WITH_PYTHON=OFF set CUDA_ARCH_NAME=All @@ -226,6 +226,8 @@ call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary set DISTUTILS_USE_SDK=1 rem Windows 10 Kit bin dir set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH% +rem Use 64-bit ToolSet to compile +set PreferredToolArchitecture=x64 for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%# set start=%start:~4,10% @@ -250,14 +252,15 @@ if "%WITH_GPU%"=="ON" ( ) rem ------initialize the python environment------ +@ECHO ON set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe set PATH=%PYTHON_ROOT%;%PYTHON_ROOT%\Scripts;%PATH% -if %WITH_PYTHON% == "OFF" ( +if "%WITH_PYTHON%" == "ON" ( where python where pip pip install wheel --user pip install -r %work_dir%\python\requirements.txt --user - if %ERRORLEVEL% NEQ 0 ( + if !ERRORLEVEL! NEQ 0 ( echo pip install requirements.txt failed! exit /b 7 ) @@ -318,7 +321,7 @@ if "%WITH_GPU%"=="ON" ( ) :cmake_impl -echo cmake .. -G %GENERATOR% -T host=x64 -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ +echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -T host=x64 -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^ -DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^ -DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^ @@ -374,6 +377,7 @@ set build_times=1 rem clcache.exe -z rem -------clean up environment again----------- +taskkill /f /im cmake.exe 2>NUL taskkill /f /im MSBuild.exe 2>NUL taskkill /f /im cl.exe 2>NUL taskkill /f /im lib.exe 2>NUL @@ -386,18 +390,20 @@ taskkill /f /im csc.exe 2>NUL taskkill /f /im nvcc.exe 2>NUL taskkill /f /im cicc.exe 2>NUL taskkill /f /im ptxas.exe 2>NUL -taskkill /f /im test_api_impl.exe 2>NUL taskkill /f /im op_function_generator.exe 2>NUL wmic process where name="op_function_generator.exe" call terminate 2>NUL -wmic process where name="test_api_impl.exe" call terminate 2>NUL wmic process where name="cvtres.exe" call terminate 2>NUL wmic process where name="rc.exe" call terminate 2>NUL -wmic process where name="CL.exe" call terminate 2>NUL -wmic process where name="Lib.exe" call terminate 2>NUL +wmic process where name="cl.exe" call terminate 2>NUL +wmic process where name="lib.exe" call terminate 2>NUL + +if "%WITH_TESTING%"=="ON" ( + for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# +) echo Build Paddle the %build_times% time: if %GENERATOR% == "Ninja" ( - ninja -j %PARALLEL_PROJECT_COUNT% + ninja all ) else ( if "%WITH_CLCACHE%"=="OFF" ( MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:TrackFileAccess=false /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj @@ -489,7 +495,6 @@ echo ======================================== echo Step 4. Running unit tests ... echo ======================================== - : set CI_SKIP_CPP_TEST if only *.py changed git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON @@ -778,15 +783,16 @@ taskkill /f /im python.exe 2>NUL taskkill /f /im nvcc.exe 2>NUL taskkill /f /im cicc.exe 2>NUL taskkill /f /im ptxas.exe 2>NUL -taskkill /f /im test_api_impl.exe 2>NUL taskkill /f /im op_function_generator.exe 2>NUL wmic process where name="op_function_generator.exe" call terminate 2>NUL -wmic process where name="test_api_impl.exe" call terminate 2>NUL wmic process where name="cvtres.exe" call terminate 2>NUL wmic process where name="rc.exe" call terminate 2>NUL -wmic process where name="CL.exe" call terminate 2>NUL -wmic process where name="Lib.exe" call terminate 2>NUL +wmic process where name="cl.exe" call terminate 2>NUL +wmic process where name="lib.exe" call terminate 2>NUL wmic process where name="python.exe" call terminate 2>NUL +if "%WITH_TESTING%"=="ON" ( + for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# +) echo Windows CI run successfully! exit /b 0 From 8e163f92afd49c301f582d2de13e3bc7cd0d1172 Mon Sep 17 00:00:00 2001 From: Wilber Date: Thu, 17 Jun 2021 15:57:55 +0800 Subject: [PATCH 121/156] [Inference Tensorrt] Add attr for trt engine and handle the input seq problem for ernie var len. (#33575) (#33622) --- .../tensorrt/convert/emb_eltwise_layernorm.cc | 2 + .../tensorrt/convert/multihead_matmul_op.cc | 12 +- .../inference/tensorrt/convert/slice_op.cc | 13 +- paddle/fluid/inference/tensorrt/engine.h | 89 +++++++++++- .../fluid/inference/tensorrt/test_engine.cc | 11 ++ .../fluid/inference/tests/api/tester_helper.h | 1 + .../tests/api/trt_dynamic_shape_ernie_test.cc | 132 ++++++++++++++++++ 7 files changed, 253 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc b/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc index 04c51202f022f..18bbd1d2b7703 100644 --- a/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc +++ b/paddle/fluid/inference/tensorrt/convert/emb_eltwise_layernorm.cc @@ -36,6 +36,8 @@ class EmbEltwiseLayerNormOpConverter : public OpConverter { framework::OpDesc op_desc(op, nullptr); auto word_id_name = op_desc.Input("WordId").front(); auto pos_id_name = op_desc.Input("PosId").front(); + engine_->Set("ernie_pos_name", new std::string(pos_id_name)); + auto sent_id_name = op_desc.Input("SentId").front(); auto word_emb_name = op_desc.Input("WordEmbedding").front(); auto pos_emb_name = op_desc.Input("PosEmbedding").front(); diff --git a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc index f2f45c694ab44..d05c9019a29d3 100644 --- a/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/multihead_matmul_op.cc @@ -191,9 +191,15 @@ class MultiheadMatMulOpConverter : public OpConverter { std::vector plugin_inputs; plugin_inputs.emplace_back(fc_layer->getOutput(0)); plugin_inputs.emplace_back(mask_tensor); - plugin_inputs.emplace_back(engine_->GetITensor( - engine_->network()->getInput(2)->getName())); // cu_seqlens, - // eval_placeholder_2 + if (engine_->Has("ernie_pos_name")) { + plugin_inputs.emplace_back( + engine_->GetITensor(engine_->Get("ernie_pos_name"))); + } else { + plugin_inputs.emplace_back(engine_->GetITensor( + engine_->network() + ->getInput(2) + ->getName())); // cu_seqlens, eval_placeholder_2 + } auto max_seqlen_tensor = engine_->GetITensor(engine_->network()->getInput(3)->getName()); auto* shuffle_layer = TRT_ENGINE_ADD_LAYER( diff --git a/paddle/fluid/inference/tensorrt/convert/slice_op.cc b/paddle/fluid/inference/tensorrt/convert/slice_op.cc index 2ab024dff327f..7f270b1f390b7 100644 --- a/paddle/fluid/inference/tensorrt/convert/slice_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/slice_op.cc @@ -76,9 +76,16 @@ class SliceOpConverter : public OpConverter { std::vector plugin_inputs; // plugin_inputs.emplace_back(trans_layer->getOutput(0)); plugin_inputs.emplace_back(input); - plugin_inputs.emplace_back(engine_->GetITensor( - engine_->network()->getInput(2)->getName())); // cu_seqlens, - // eval_placeholder_2 + + std::string pos_name; + if (engine_->Has("ernie_pos_name")) { + pos_name = engine_->Get("ernie_pos_name"); + } else { + // hard code for compatibility + pos_name = engine_->network()->getInput(2)->getName(); + } + plugin_inputs.emplace_back( + engine_->GetITensor(pos_name)); // cu_seqlens, eval_placeholder_2 // bool ban_fp16 = engine_->disable_trt_plugin_fp16(); plugin::SpecialSlicePluginDynamic* plugin = diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h index 2358e1ef976cd..7e5707269782e 100644 --- a/paddle/fluid/inference/tensorrt/engine.h +++ b/paddle/fluid/inference/tensorrt/engine.h @@ -202,7 +202,15 @@ class TensorRTEngine { dy::initLibNvInferPlugins(&logger, ""); } - ~TensorRTEngine() {} + ~TensorRTEngine() { + for (auto& attr : attrs_) { + if (attr_dels_.find(attr.first) != attr_dels_.end()) { + attr_dels_[attr.first](); + } + } + attrs_.clear(); + attr_dels_.clear(); + } // Add an input and set its name, data type and dimension. nvinfer1::ITensor* DeclareInput(const std::string& name, @@ -386,6 +394,82 @@ class TensorRTEngine { } #endif + bool Has(const std::string& attr_name) const { + return attrs_.count(attr_name) > 0; + } + + void Erase(const std::string& attr_name) { + if (!Has(attr_name)) { + return; + } + if (attr_dels_.find(attr_name) != attr_dels_.end()) { + attr_dels_[attr_name](); + attr_dels_.erase(attr_name); + } + attrs_.erase(attr_name); + } + + // Set a pointer to the attribute. Engine takes ownership of the attribute. + template + void Set(const std::string& attr_name, AttrType* attr) { + if (attrs_.count(attr_name) == 0) { + PADDLE_ENFORCE_EQ( + attrs_.count(attr_name), 0, + platform::errors::AlreadyExists( + "Attribute %s already set in trt engine.", attr_name)); + } else { + VLOG(3) << "Setting the attribute " << attr_name << " for trt engine " + << this; + } + attrs_[attr_name] = attr; + attr_dels_[attr_name] = [attr, attr_name]() { + VLOG(3) << "deleting " << attr_name; + delete attr; + }; + } + + // Set a pointer to the attribute. Engine doesn't take ownership. Caller + // should delete the attribute. + template + void SetNotOwned(const std::string& attr_name, AttrType* attr) { + PADDLE_ENFORCE_EQ( + attrs_.count(attr_name), 0, + platform::errors::AlreadyExists( + "Attribute %s already set in trt engine.", attr_name)); + attrs_[attr_name] = attr; + } + + // Get a reference to the attributed previously set. + template + AttrType& Get(const std::string& attr_name) const { + PADDLE_ENFORCE_NE(attrs_.find(attr_name), attrs_.end(), + platform::errors::InvalidArgument( + "Attribute %s not found in trt engine.", attr_name)); + try { + return *boost::any_cast(attrs_.at(attr_name)); + } catch (boost::bad_any_cast&) { + auto TypeToString = [](const std::type_info& info) -> std::string { + if (std::type_index(info) == std::type_index(typeid(bool*))) { + return "bool"; + } else if (std::type_index(info) == std::type_index(typeid(int*))) { + return "int"; + } else if (std::type_index(info) == + std::type_index(typeid(const int*))) { + return "const int"; + } else if (std::type_index(info) == + std::type_index(typeid(std::string*))) { + return "std::string"; + } + return info.name(); + }; + + PADDLE_THROW(platform::errors::InvalidArgument( + "Invalid type for attritube %s, expected: %s, actual: %s.", attr_name, + TypeToString(typeid(AttrType*)), + TypeToString(attrs_.at(attr_name).type()))); + } + } + private: // Each ICudaEngine object is bound to a specific GPU when it is instantiated, // ensure that the thread is associated with the correct device by calling @@ -441,6 +525,9 @@ class TensorRTEngine { infer_ptr ihost_memory_; std::unordered_map quant_dynamic_range_; + std::unordered_map attrs_; + std::unordered_map> attr_dels_; + // For dynamic shape bool with_dynamic_shape_{false}; infer_ptr infer_networkv2_; diff --git a/paddle/fluid/inference/tensorrt/test_engine.cc b/paddle/fluid/inference/tensorrt/test_engine.cc index 7c763858bb210..5c61bec55ba71 100644 --- a/paddle/fluid/inference/tensorrt/test_engine.cc +++ b/paddle/fluid/inference/tensorrt/test_engine.cc @@ -91,6 +91,15 @@ TEST_F(TensorRTEngineTest, add_layer) { buffers[0] = reinterpret_cast(x_v_gpu_data); buffers[1] = reinterpret_cast(y_gpu_data); + LOG(INFO) << "Set attr"; + engine_->Set("test_attr", new std::string("test_attr")); + if (engine_->Has("test_attr")) { + auto attr_val = engine_->Get("test_attr"); + engine_->Erase("test_attr"); + } + std::string *attr_key = new std::string("attr_key"); + engine_->SetNotOwned("attr1", attr_key); + LOG(INFO) << "to execute"; engine_->Execute(1, &buffers, ctx_->stream()); @@ -99,6 +108,8 @@ TEST_F(TensorRTEngineTest, add_layer) { LOG(INFO) << "to checkout output"; ASSERT_EQ(y_cpu[0], x_v[0] * 2 + 3); + + delete attr_key; } TEST_F(TensorRTEngineTest, add_layer_multi_dim) { diff --git a/paddle/fluid/inference/tests/api/tester_helper.h b/paddle/fluid/inference/tests/api/tester_helper.h index 170b915ec7436..dbc2acbed8367 100644 --- a/paddle/fluid/inference/tests/api/tester_helper.h +++ b/paddle/fluid/inference/tests/api/tester_helper.h @@ -33,6 +33,7 @@ #include "paddle/fluid/inference/analysis/ut_helper.h" #include "paddle/fluid/inference/api/analysis_predictor.h" #include "paddle/fluid/inference/api/helper.h" +#include "paddle/fluid/inference/api/paddle_inference_api.h" #include "paddle/fluid/inference/api/paddle_inference_pass.h" #include "paddle/fluid/inference/tests/api/config_printer.h" #include "paddle/fluid/inference/tests/test_helper.h" diff --git a/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc b/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc index 6d69565716ee7..45dff9f4c3710 100644 --- a/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc +++ b/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc @@ -16,6 +16,7 @@ limitations under the License. */ #include #include "gflags/gflags.h" +#include "paddle/fluid/inference/tensorrt/helper.h" #include "paddle/fluid/inference/tests/api/trt_test_helper.h" namespace paddle { @@ -143,5 +144,136 @@ TEST(AnalysisPredictor, fp16) { #endif } +// ernie_varlen +std::shared_ptr InitPredictor() { + paddle_infer::Config config; + config.SetModel(FLAGS_infer_model); + + config.EnableUseGpu(100, 0); + + // Open the memory optim. + config.EnableMemoryOptim(); + + int max_batch = 32; + int max_single_seq_len = 128; + int opt_single_seq_len = 64; + int min_batch_seq_len = 1; + int max_batch_seq_len = 512; + int opt_batch_seq_len = 256; + + std::string input_name0 = "read_file_0.tmp_0"; + std::string input_name1 = "read_file_0.tmp_1"; + std::string input_name2 = "read_file_0.tmp_2"; + std::string input_name3 = "read_file_0.tmp_4"; + + std::vector min_shape = {min_batch_seq_len}; + std::vector max_shape = {max_batch_seq_len}; + std::vector opt_shape = {opt_batch_seq_len}; + // Set the input's min, max, opt shape + std::map> min_input_shape = { + {input_name0, min_shape}, + {input_name1, min_shape}, + {input_name2, {1}}, + {input_name3, {1, 1, 1}}}; + std::map> max_input_shape = { + {input_name0, max_shape}, + {input_name1, max_shape}, + {input_name2, {max_batch + 1}}, + {input_name3, {1, max_single_seq_len, 1}}}; + std::map> opt_input_shape = { + {input_name0, opt_shape}, + {input_name1, opt_shape}, + {input_name2, {max_batch + 1}}, + {input_name3, {1, opt_single_seq_len, 1}}}; + + // only kHalf supported + config.EnableTensorRtEngine( + 1 << 30, 1, 5, paddle_infer::Config::Precision::kHalf, false, false); + // erinie varlen must be used with dynamic shape + config.SetTRTDynamicShapeInfo(min_input_shape, max_input_shape, + opt_input_shape); + // erinie varlen must be used with oss + config.EnableTensorRtOSS(); + + return paddle_infer::CreatePredictor(config); +} + +void run(paddle_infer::Predictor* predictor, std::vector* out_data) { + const int run_batch = 2; + const int run_seq_len = 71; + const int max_seq_len = 128; + + int32_t i1[run_seq_len] = { + // sentence 1 + 1, 3558, 4, 75, 491, 89, 340, 313, 93, 4, 255, 10, 75, 321, 4095, 1902, 4, + 134, 49, 75, 311, 14, 44, 178, 543, 15, 12043, 2, 75, 201, 340, 9, 14, 44, + 486, 218, 1140, 279, 12043, 2, + // sentence 2 + 101, 2054, 2234, 2046, 2486, 2044, 1996, 2047, 4552, 2001, 9536, 1029, + 102, 2004, 1997, 2008, 2154, 1010, 1996, 2047, 4552, 9536, 2075, 1996, + 2117, 3072, 2234, 2046, 2486, 1012, 102, + }; + int32_t i2[run_seq_len] = { + // sentence 1 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + // sentence 2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1}; + // shape info of this batch + int32_t i3[3] = {0, 40, 71}; + // max_seq_len represents the max sentence length of all the sentences, only + // length of + // input i4 is useful, data means nothing. + int32_t i4[max_seq_len] = {0}; + + auto input_names = predictor->GetInputNames(); + // first input + auto input_t1 = predictor->GetInputHandle(input_names[0]); + input_t1->Reshape({run_seq_len}); + input_t1->CopyFromCpu(i1); + + // second input + auto input_t2 = predictor->GetInputHandle(input_names[1]); + input_t2->Reshape({run_seq_len}); + input_t2->CopyFromCpu(i2); + + // third input + auto input_t3 = predictor->GetInputHandle(input_names[2]); + input_t3->Reshape({run_batch + 1}); + input_t3->CopyFromCpu(i3); + + // fourth input + auto input_t4 = predictor->GetInputHandle(input_names[3]); + input_t4->Reshape({1, max_seq_len, 1}); + input_t4->CopyFromCpu(i4); + + CHECK(predictor->Run()); + + auto output_names = predictor->GetOutputNames(); + auto output_t = predictor->GetOutputHandle(output_names[0]); + std::vector output_shape = output_t->shape(); + int out_num = std::accumulate(output_shape.begin(), output_shape.end(), 1, + std::multiplies()); + out_data->resize(out_num); + output_t->CopyToCpu(out_data->data()); + + return; +} + +TEST(AnalysisPredictor, ernie_varlen) { +#if IS_TRT_VERSION_GE(7234) + auto predictor = InitPredictor(); + std::vector out_data; + run(predictor.get(), &out_data); + std::vector ref_data{0.59814, 0.219882, 0.181978, + 0.359796, 0.577414, 0.0627908}; + float near_tolerance = 1e-3; + for (size_t i = 0; i < out_data.size(); i++) { + EXPECT_NEAR(ref_data[i], out_data[i], near_tolerance); + } +#endif +} + } // namespace inference } // namespace paddle From 40b2a034474f919824535f2a209f578892ea422f Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Fri, 18 Jun 2021 11:07:45 +0800 Subject: [PATCH 122/156] [cherry-pick 32784] Fix distro (#33638) cherry-pick 32784 --- paddle/scripts/paddle_build.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index cec7f6ef50abf..2af767472face 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -248,6 +248,12 @@ function cmake_base() { distibuted_flag=${WITH_DISTRIBUTE:-OFF} gloo_flag=${distibuted_flag} + if [ "$CMD" != "assert_file_approvals" ];then + python -m pip install distro + python ${PADDLE_ROOT}/tools/summary_env.py + bash ${PADDLE_ROOT}/tools/get_cpu_info.sh + fi + cat < Date: Fri, 18 Jun 2021 11:09:22 +0800 Subject: [PATCH 123/156] remove check for optim_cache_dir in trt slim int8 (#32676) (#33629) --- paddle/fluid/inference/analysis/ir_pass_manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/inference/analysis/ir_pass_manager.cc b/paddle/fluid/inference/analysis/ir_pass_manager.cc index 8407f98e6dfd9..4bb08dc96b1cf 100644 --- a/paddle/fluid/inference/analysis/ir_pass_manager.cc +++ b/paddle/fluid/inference/analysis/ir_pass_manager.cc @@ -106,8 +106,8 @@ void IRPassManager::CreatePasses(Argument *argument, bool use_static_engine = argument->tensorrt_use_static_engine(); bool model_from_memory = argument->model_from_memory(); std::string optim_cache_dir = argument->optim_cache_dir(); - bool int8_valid = - !(model_from_memory && optim_cache_dir.empty() && enable_int8); + bool int8_valid = !(model_from_memory && optim_cache_dir.empty() && + enable_int8 && use_calib_mode); PADDLE_ENFORCE_EQ( int8_valid, true, platform::errors::PreconditionNotMet( From 6ec2ea0f945243b0f1e5b53cd33751f4bbf07177 Mon Sep 17 00:00:00 2001 From: wuhuanzhou Date: Fri, 18 Jun 2021 11:32:47 +0800 Subject: [PATCH 124/156] [cherry-pick] fix cmake expressions error #33621 cherry-pick #32815 --- cmake/external/warpctc.cmake | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 100b915339469..c591a9391dfa5 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -78,6 +78,21 @@ if(WITH_ASCEND OR WITH_ASCEND_CL) -DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR} ) else() + if(WIN32) + set(WARPCTC_C_FLAGS $) + set(WARPCTC_C_FLAGS_DEBUG $) + set(WARPCTC_C_FLAGS_RELEASE $) + set(WARPCTC_CXX_FLAGS $) + set(WARPCTC_CXX_FLAGS_RELEASE $) + set(WARPCTC_CXX_FLAGS_DEBUG $) + else() + set(WARPCTC_C_FLAGS ${CMAKE_C_FLAGS}) + set(WARPCTC_C_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) + set(WARPCTC_C_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE}) + set(WARPCTC_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + set(WARPCTC_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + set(WARPCTC_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + endif() ExternalProject_Add( extern_warpctc ${EXTERNAL_PROJECT_LOG_ARGS} @@ -90,12 +105,12 @@ else() BUILD_ALWAYS 1 CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_C_FLAGS=$ - -DCMAKE_C_FLAGS_DEBUG=$ - -DCMAKE_C_FLAGS_RELEASE=$ - -DCMAKE_CXX_FLAGS=$ - -DCMAKE_CXX_FLAGS_RELEASE=$ - -DCMAKE_CXX_FLAGS_DEBUG=$ + -DCMAKE_C_FLAGS=${WARPCTC_C_FLAGS} + -DCMAKE_C_FLAGS_DEBUG=${WARPCTC_C_FLAGS_DEBUG} + -DCMAKE_C_FLAGS_RELEASE=${WARPCTC_C_FLAGS_RELEASE} + -DCMAKE_CXX_FLAGS=${WARPCTC_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_RELEASE=${WARPCTC_CXX_FLAGS_RELEASE} + -DCMAKE_CXX_FLAGS_DEBUG=${WARPCTC_CXX_FLAGS_DEBUG} -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} -DWITH_GPU=${WITH_GPU} -DWITH_ROCM=${WITH_ROCM} From bd3aa038eded5b4661d9aca021cec8bacad7a46a Mon Sep 17 00:00:00 2001 From: Wilber Date: Fri, 18 Jun 2021 15:22:52 +0800 Subject: [PATCH 125/156] [XPU] Update cmake options for xpu. (#33450) (#33581) * [XPU] Update cmake options for xpu. (#33450) * patch in inference third_party --- cmake/external/lite.cmake | 30 ++++++--- cmake/external/xpu.cmake | 127 +++++++++++++++++++++----------------- cmake/inference_lib.cmake | 7 +++ python/setup.py.in | 12 ---- 4 files changed, 101 insertions(+), 75 deletions(-) diff --git a/cmake/external/lite.cmake b/cmake/external/lite.cmake index 6e2157e308716..e213068377b14 100644 --- a/cmake/external/lite.cmake +++ b/cmake/external/lite.cmake @@ -18,13 +18,21 @@ if(NOT LINUX) return() endif() -if(XPU_SDK_ROOT) - set(LITE_WITH_XPU ON) - include_directories("${XPU_SDK_ROOT}/XTDK/include") - include_directories("${XPU_SDK_ROOT}/XTCL/include") +if (LITE_WITH_XPU) add_definitions(-DLITE_SUBGRAPH_WITH_XPU) - LINK_DIRECTORIES("${XPU_SDK_ROOT}/XTDK/shlib/") - LINK_DIRECTORIES("${XPU_SDK_ROOT}/XTDK/runtime/shlib/") + IF(WITH_AARCH64) + SET(XPU_SDK_ENV "kylin_aarch64") + ELSEIF(WITH_SUNWAY) + SET(XPU_SDK_ENV "deepin_sw6_64") + ELSEIF(WITH_BDCENTOS) + SET(XPU_SDK_ENV "bdcentos_x86_64") + ELSEIF(WITH_UBUNTU) + SET(XPU_SDK_ENV "ubuntu_x86_64") + ELSEIF(WITH_CENTOS) + SET(XPU_SDK_ENV "centos7_x86_64") + ELSE () + SET(XPU_SDK_ENV "ubuntu_x86_64") + ENDIF() endif() if (NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR) @@ -57,7 +65,8 @@ if (NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR) -DWITH_TESTING=OFF -DLITE_BUILD_EXTRA=ON -DLITE_WITH_XPU=${LITE_WITH_XPU} - -DXPU_SDK_ROOT=${XPU_SDK_ROOT} + -DXPU_SDK_URL=${XPU_BASE_URL} + -DXPU_SDK_ENV=${XPU_SDK_ENV} -DLITE_WITH_CODE_META_INFO=OFF -DLITE_WITH_ARM=ON) ExternalProject_Add( @@ -99,7 +108,8 @@ if (NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR) -DLITE_WITH_STATIC_CUDA=OFF -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME} -DLITE_WITH_XPU=${LITE_WITH_XPU} - -DXPU_SDK_ROOT=${XPU_SDK_ROOT} + -DXPU_SDK_URL=${XPU_BASE_URL} + -DXPU_SDK_ENV=${XPU_SDK_ENV} -DLITE_WITH_CODE_META_INFO=OFF -DLITE_WITH_ARM=OFF) @@ -147,6 +157,10 @@ message(STATUS "Paddle-lite BINARY_DIR: ${LITE_BINARY_DIR}") message(STATUS "Paddle-lite SOURCE_DIR: ${LITE_SOURCE_DIR}") include_directories(${LITE_SOURCE_DIR}) include_directories(${LITE_BINARY_DIR}) +if(LITE_WITH_XPU) + include_directories(${LITE_BINARY_DIR}/third_party/install/xpu/xdnn/include/) + include_directories(${LITE_BINARY_DIR}/third_party/install/xpu/xre/include/) +endif() function(external_lite_libs alias path) add_library(${alias} SHARED IMPORTED GLOBAL) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index a03ff7d22dcad..a8c33618a6135 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -7,52 +7,73 @@ SET(XPU_PROJECT "extern_xpu") SET(XPU_API_LIB_NAME "libxpuapi.so") SET(XPU_RT_LIB_NAME "libxpurt.so") -if(NOT XPU_SDK_ROOT) - if (WITH_AARCH64) - SET(XPU_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/aarch64/xpu_2021_01_13.tar.gz" CACHE STRING "" FORCE) - elseif(WITH_SUNWAY) - SET(XPU_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/sunway/xpu_2021_01_13.tar.gz" CACHE STRING "" FORCE) - else() - SET(XPU_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/xpu_2021_05_19.tar.gz" CACHE STRING "" FORCE) - endif() - - SET(XPU_SOURCE_DIR "${THIRD_PARTY_PATH}/xpu") - SET(XPU_DOWNLOAD_DIR "${XPU_SOURCE_DIR}/src/${XPU_PROJECT}") - SET(XPU_INSTALL_DIR "${THIRD_PARTY_PATH}/install/xpu") - SET(XPU_API_INC_DIR "${THIRD_PARTY_PATH}/install/xpu/include") - SET(XPU_LIB_DIR "${THIRD_PARTY_PATH}/install/xpu/lib") - - SET(XPU_API_LIB "${XPU_LIB_DIR}/${XPU_API_LIB_NAME}") - SET(XPU_RT_LIB "${XPU_LIB_DIR}/${XPU_RT_LIB_NAME}") - - SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${XPU_INSTALL_DIR}/lib") - - FILE(WRITE ${XPU_DOWNLOAD_DIR}/CMakeLists.txt - "PROJECT(XPU)\n" - "cmake_minimum_required(VERSION 3.0)\n" - "install(DIRECTORY xpu/include xpu/lib \n" - " DESTINATION ${XPU_INSTALL_DIR})\n") - - ExternalProject_Add( - ${XPU_PROJECT} - ${EXTERNAL_PROJECT_LOG_ARGS} - PREFIX ${XPU_SOURCE_DIR} - DOWNLOAD_DIR ${XPU_DOWNLOAD_DIR} - DOWNLOAD_COMMAND wget --no-check-certificate ${XPU_URL} -c -q -O xpu.tar.gz - && tar xvf xpu.tar.gz - DOWNLOAD_NO_PROGRESS 1 - UPDATE_COMMAND "" - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${XPU_INSTALL_ROOT} - CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${XPU_INSTALL_ROOT} - ) -else() - SET(XPU_API_INC_DIR "${XPU_SDK_ROOT}/XTDK/include/") - SET(XPU_API_LIB "${XPU_SDK_ROOT}/XTDK/shlib/libxpuapi.so") - SET(XPU_RT_LIB "${XPU_SDK_ROOT}/XTDK/runtime/shlib/libxpurt.so") - SET(XPU_LIB_DIR "${XPU_SDK_ROOT}/XTDK/shlib/") -endif() +IF(WITH_AARCH64) + SET(XPU_XRE_DIR_NAME "xre-kylin_aarch64") + SET(XPU_XDNN_DIR_NAME "xdnn-kylin_aarch64") + SET(XPU_XCCL_DIR_NAME "xccl-kylin_aarch64") +ELSEIF(WITH_SUNWAY) + SET(XPU_XRE_DIR_NAME "xre-deepin_sw6_64") + SET(XPU_XDNN_DIR_NAME "xdnn-deepin_sw6_64") + SET(XPU_XCCL_DIR_NAME "xccl-deepin_sw6_64") +ELSEIF(WITH_BDCENTOS) + SET(XPU_XRE_DIR_NAME "xre-bdcentos_x86_64") + SET(XPU_XDNN_DIR_NAME "xdnn-bdcentos_x86_64") + SET(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64") +ELSEIF(WITH_UBUNTU) + SET(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64") + SET(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64") + SET(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64") +ELSEIF(WITH_CENTOS) + SET(XPU_XRE_DIR_NAME "xre-centos7_x86_64") + SET(XPU_XDNN_DIR_NAME "xdnn-centos7_x86_64") + SET(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64") +ELSE () + SET(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64") + SET(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64") + SET(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64") +ENDIF() + +IF(NOT XPU_BASE_URL) + SET(XPU_BASE_URL "https://baidu-kunlun-product.cdn.bcebos.com/KL-SDK/klsdk-dev/20210527") +ENDIF() + +SET(XPU_XRE_URL "${XPU_BASE_URL}/${XPU_XRE_DIR_NAME}.tar.gz" CACHE STRING "" FORCE) +SET(XPU_XDNN_URL "${XPU_BASE_URL}/${XPU_XDNN_DIR_NAME}.tar.gz" CACHE STRING "" FORCE) +SET(XPU_XCCL_URL "${XPU_BASE_URL}/${XPU_XCCL_DIR_NAME}.tar.gz" CACHE STRING "" FORCE) +SET(XPU_PACK_DEPENCE_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/pack_paddle_depence.sh" CACHE STRING "" FORCE) + +SET(XPU_SOURCE_DIR "${THIRD_PARTY_PATH}/xpu") +SET(XPU_DOWNLOAD_DIR "${XPU_SOURCE_DIR}/src/${XPU_PROJECT}") +SET(XPU_INSTALL_DIR "${THIRD_PARTY_PATH}/install/xpu") +SET(XPU_INC_DIR "${THIRD_PARTY_PATH}/install/xpu/include") +SET(XPU_LIB_DIR "${THIRD_PARTY_PATH}/install/xpu/lib") + +SET(XPU_API_LIB "${XPU_LIB_DIR}/${XPU_API_LIB_NAME}") +SET(XPU_RT_LIB "${XPU_LIB_DIR}/${XPU_RT_LIB_NAME}") + +SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${XPU_INSTALL_DIR}/lib") -INCLUDE_DIRECTORIES(${XPU_API_INC_DIR}) +FILE(WRITE ${XPU_DOWNLOAD_DIR}/CMakeLists.txt + "PROJECT(XPU)\n" + "cmake_minimum_required(VERSION 3.0)\n" + "install(DIRECTORY xpu/include xpu/lib \n" + " DESTINATION ${XPU_INSTALL_DIR})\n") + +ExternalProject_Add( + ${XPU_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${XPU_SOURCE_DIR} + DOWNLOAD_DIR ${XPU_DOWNLOAD_DIR} + DOWNLOAD_COMMAND wget ${XPU_PACK_DEPENCE_URL} + && bash pack_paddle_depence.sh ${XPU_XRE_URL} ${XPU_XRE_DIR_NAME} ${XPU_XDNN_URL} ${XPU_XDNN_DIR_NAME} ${XPU_XCCL_URL} ${XPU_XCCL_DIR_NAME} + + DOWNLOAD_NO_PROGRESS 1 + UPDATE_COMMAND "" + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${XPU_INSTALL_ROOT} + CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${XPU_INSTALL_ROOT} +) + +INCLUDE_DIRECTORIES(${XPU_INC_DIR}) ADD_LIBRARY(shared_xpuapi SHARED IMPORTED GLOBAL) set_property(TARGET shared_xpuapi PROPERTY IMPORTED_LOCATION "${XPU_API_LIB}") @@ -62,7 +83,7 @@ generate_dummy_static_lib(LIB_NAME "xpulib" GENERATOR "xpu.cmake") TARGET_LINK_LIBRARIES(xpulib ${XPU_API_LIB} ${XPU_RT_LIB}) -if (WITH_XPU_BKCL) +IF(WITH_XPU_BKCL) MESSAGE(STATUS "Compile with XPU BKCL!") ADD_DEFINITIONS(-DPADDLE_WITH_XPU_BKCL) @@ -71,15 +92,11 @@ if (WITH_XPU_BKCL) SET(XPU_BKCL_INC_DIR "${THIRD_PARTY_PATH}/install/xpu/include") INCLUDE_DIRECTORIES(${XPU_BKCL_INC_DIR}) TARGET_LINK_LIBRARIES(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ${XPU_BKCL_LIB}) -else(WITH_XPU_BKCL) - TARGET_LINK_LIBRARIES(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ) -endif(WITH_XPU_BKCL) - -if(NOT XPU_SDK_ROOT) - ADD_DEPENDENCIES(xpulib ${XPU_PROJECT}) -else() - ADD_CUSTOM_TARGET(extern_xpu DEPENDS xpulib) -endif() +ELSE(WITH_XPU_BKCL) + TARGET_LINK_LIBRARIES(xpulib ${XPU_API_LIB} ${XPU_RT_LIB}) +ENDIF(WITH_XPU_BKCL) + +ADD_DEPENDENCIES(xpulib ${XPU_PROJECT}) # Ensure that xpu/api.h can be included without dependency errors. file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/.xpu_headers_dummy.cc CONTENT "") diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index a10b5b231c875..9574af761ed10 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -154,6 +154,13 @@ IF(WITH_GPU) DSTS ${dst_dir}) ENDIF() +IF(WITH_XPU) + set(dst_dir "${PADDLE_INFERENCE_INSTALL_DIR}/third_party/install/xpu") + copy(inference_lib_dist + SRCS ${XPU_INC_DIR} ${XPU_LIB_DIR} + DSTS ${dst_dir} ${dst_dir}) +ENDIF() + # CMakeCache Info copy(inference_lib_dist SRCS ${CMAKE_CURRENT_BINARY_DIR}/CMakeCache.txt diff --git a/python/setup.py.in b/python/setup.py.in index 0f2e97192c1df..6787a524d7a87 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -345,18 +345,6 @@ if '${WITH_XPU_BKCL}' == 'ON': shutil.copy('${XPU_BKCL_LIB}', libs_path) package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}'] -# Only for lite xpu inference. -if '${WITH_XPU}' == 'OFF' and '${XPU_SDK_ROOT}' != '': - xpu_api_lib = os.path.join('${XPU_SDK_ROOT}', 'XTDK/shlib/', 'libxpuapi.so') - xpu_rt_lib = os.path.join('${XPU_SDK_ROOT}', 'XTDK/runtime/shlib/', 'libxpurt.so') - if os.path.exists(xpu_api_lib): - shutil.copy(xpu_api_lib, libs_path) - package_data['paddle.libs']+=['libxpuapi.so'] - if os.path.exists(xpu_rt_lib): - shutil.copy(xpu_rt_lib, libs_path) - package_data['paddle.libs']+=['libxpurt.so'] - - # remove unused paddle/libs/__init__.py if os.path.isfile(libs_path+'/__init__.py'): os.remove(libs_path+'/__init__.py') From 9a3d859390c233afa5ce0baf8cfceb182d89025e Mon Sep 17 00:00:00 2001 From: Wilber Date: Fri, 18 Jun 2021 15:23:18 +0800 Subject: [PATCH 126/156] cherry-pick .Align the code of trt under the develop and release/2.1 branch (#33631) --- .../tensorrt/convert/elementwise_op.cc | 31 ++- .../fluid/inference/tensorrt/convert/fc_op.cc | 247 ++++++------------ .../inference/tensorrt/convert/flatten_op.cc | 55 +++- .../inference/tensorrt/convert/reshape_op.cc | 2 +- paddle/fluid/inference/tensorrt/op_teller.cc | 53 ++-- 5 files changed, 187 insertions(+), 201 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc index 5419933e40736..df2400854414c 100644 --- a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc @@ -62,6 +62,25 @@ class ElementwiseWeightOpConverter : public OpConverter { 0}; TensorRTEngine::Weight power_weights{nvinfer1::DataType::kFLOAT, nullptr, 0}; + + nvinfer1::IShuffleLayer* expand_layer = nullptr; + nvinfer1::IShuffleLayer* squeeze_layer = nullptr; + int dynamic_shape_offset = engine_->with_dynamic_shape() ? 1 : 0; + auto input_dim = X->getDimensions(); + if (input_dim.nbDims < 3 + dynamic_shape_offset) { + nvinfer1::Dims expand_shape; + expand_shape.nbDims = 3 + dynamic_shape_offset; + for (int i = 0; i < expand_shape.nbDims; i++) { + if (i < input_dim.nbDims) { + expand_shape.d[i] = input_dim.d[i] < 0 ? 0 : input_dim.d[i]; + } else { + expand_shape.d[i] = 1; + } + } + expand_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *X); + expand_layer->setReshapeDimensions(expand_shape); + X = expand_layer->getOutput(0); + } if (op_type_ == "add") { nvinfer1::IScaleLayer* scale_layer = TRT_ENGINE_ADD_LAYER( engine_, Scale, *X, scale_mode, shift_weights.get(), @@ -73,7 +92,17 @@ class ElementwiseWeightOpConverter : public OpConverter { shift_weights.get(), power_weights.get()); layer = scale_layer; } - + if (input_dim.nbDims < 3 + dynamic_shape_offset) { + nvinfer1::Dims squeeze_shape; + squeeze_shape.nbDims = input_dim.nbDims; + for (int i = 0; i < squeeze_shape.nbDims; i++) { + squeeze_shape.d[i] = input_dim.d[i] < 0 ? 0 : input_dim.d[i]; + } + squeeze_layer = + TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *(layer->getOutput(0))); + squeeze_layer->setReshapeDimensions(squeeze_shape); + layer = static_cast(squeeze_layer); + } auto output_name = op_desc.Output("Out")[0]; RreplenishLayerAndOutput(layer, "elementwise_" + op_type_, {output_name}, test_mode); diff --git a/paddle/fluid/inference/tensorrt/convert/fc_op.cc b/paddle/fluid/inference/tensorrt/convert/fc_op.cc index 6167e68df2b67..74bb854e55f82 100644 --- a/paddle/fluid/inference/tensorrt/convert/fc_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/fc_op.cc @@ -37,7 +37,7 @@ class FcOpConverter : public OpConverter { const framework::Scope& scope, bool test_mode) override { VLOG(3) << "convert a fluid fc op to tensorrt fc layer without bias"; framework::OpDesc op_desc(op, nullptr); - + auto output_name = op_desc.Output("Out").front(); auto input_names = op_desc.InputNames(); bool with_bias = input_names.size() >= 3; std::string w_name = "Y"; @@ -48,13 +48,14 @@ class FcOpConverter : public OpConverter { } // Declare inputs auto* X = engine_->GetITensor(op_desc.Input(i_name).front()); + auto x_dim = X->getDimensions(); // Declare weights auto* Y_v = scope.FindVar(op_desc.Input(w_name).front()); PADDLE_ENFORCE_NOT_NULL( Y_v, platform::errors::NotFound( "Can not find %s presistale var of fc in scope.", w_name)); auto* Y_t = Y_v->GetMutable(); - const int x_num_col_dims = + int x_num_col_dims = op_desc.HasAttr("x_num_col_dims") ? BOOST_GET_CONST(int, op_desc.GetAttr("x_num_col_dims")) : (op_desc.HasAttr("in_num_col_dims") @@ -106,8 +107,8 @@ class FcOpConverter : public OpConverter { auto regist_fc = [&](nvinfer1::ITensor* inputs, int n_output, TensorRTEngine::Weight& weight, TensorRTEngine::Weight& bias) { - nvinfer1::ILayer* fc_layer = nullptr; if (enable_int8) { + // add conv layer PADDLE_ENFORCE_EQ( op_desc.HasAttr("out_threshold"), true, platform::errors::InvalidArgument( @@ -115,22 +116,52 @@ class FcOpConverter : public OpConverter { float out_scale = BOOST_GET_CONST(float, op_desc.GetAttr("out_threshold")); nvinfer1::DimsHW nv_ksize(1, 1); - fc_layer = TRT_ENGINE_ADD_LAYER(engine_, Convolution, *inputs, n_output, - nv_ksize, weight.get(), bias.get()); - engine_->SetTensorDynamicRange(fc_layer->getOutput(0), out_scale); - } else { - fc_layer = TRT_ENGINE_ADD_LAYER(engine_, FullyConnected, *inputs, - n_output, weight.get(), bias.get()); - } - - auto output_name = op_desc.Output("Out").front(); - if (activation_type == "relu") { - nvinfer1::IActivationLayer* relu_layer = - TRT_ENGINE_ADD_LAYER(engine_, Activation, *(fc_layer->getOutput(0)), - nvinfer1::ActivationType::kRELU); - RreplenishLayerAndOutput(relu_layer, "fc", {output_name}, test_mode); + auto* fc_layer_int8 = + TRT_ENGINE_ADD_LAYER(engine_, Convolution, *inputs, n_output, + nv_ksize, weight.get(), bias.get()); + engine_->SetTensorDynamicRange(fc_layer_int8->getOutput(0), out_scale); + if (activation_type == "relu") { + nvinfer1::IActivationLayer* relu_layer_int8 = TRT_ENGINE_ADD_LAYER( + engine_, Activation, *(fc_layer_int8->getOutput(0)), + nvinfer1::ActivationType::kRELU); + RreplenishLayerAndOutput(relu_layer_int8, "relu_after_fc_shuffle", + {output_name}, test_mode); + } else { + RreplenishLayerAndOutput(fc_layer_int8, "shuffle_after_fc", + {output_name}, test_mode); + } } else { - RreplenishLayerAndOutput(fc_layer, "fc", {output_name}, test_mode); + // add fc layer + auto* fc_layer_before = + TRT_ENGINE_ADD_LAYER(engine_, FullyConnected, *inputs, n_output, + weight.get(), bias.get()); + fc_layer_before->setName( + ("fc_layer_before(Output: " + output_name + ")").c_str()); + // add shuffle after fc + nvinfer1::Dims reshape_after_fc_dim; + if (engine_->use_oss() && engine_->with_ernie() && x_dim.nbDims == 4 && + x_dim.d[2] == 1 && x_dim.d[3] == 1 && x_num_col_dims == 1) { + // If use tensorrt'oss, the x_dim and x_num_col_dims need change + reshape_after_fc_dim.nbDims = 4; + } else { + reshape_after_fc_dim.nbDims = x_num_col_dims + 1; + } + for (int i = 0; i < reshape_after_fc_dim.nbDims; i++) { + reshape_after_fc_dim.d[i] = 0; + } + auto* fc_layer_float = TRT_ENGINE_ADD_LAYER( + engine_, Shuffle, *fc_layer_before->getOutput(0)); + fc_layer_float->setReshapeDimensions(reshape_after_fc_dim); + if (activation_type == "relu") { + nvinfer1::IActivationLayer* relu_layer_float = TRT_ENGINE_ADD_LAYER( + engine_, Activation, *(fc_layer_float->getOutput(0)), + nvinfer1::ActivationType::kRELU); + RreplenishLayerAndOutput(relu_layer_float, "relu_after_fc_shuffle", + {output_name}, test_mode); + } else { + RreplenishLayerAndOutput(fc_layer_float, "shuffle_after_fc", + {output_name}, test_mode); + } } }; @@ -157,153 +188,47 @@ class FcOpConverter : public OpConverter { static_cast(bias_data), static_cast(bias_num)}; - if (engine_->with_dynamic_shape()) { - // not NCHW layout, but NLP layout with added 'x 1 x 1' - auto x_dim = X->getDimensions(); - if (engine_->use_oss() && engine_->with_ernie() && x_dim.nbDims == 4 && - x_dim.d[2] == 1 && x_dim.d[3] == 1 && x_num_col_dims == 2) { - // fc which is just after self attention - regist_fc(X, n_output, weight, bias); - return; - } - PADDLE_ENFORCE_LE( - x_dim.nbDims - x_num_col_dims, 3, - platform::errors::InvalidArgument( - "Params and input dims mismatch. Paddle-TRT FC " - "converter expects x_dim.nbDims - x_num_col_dims <= 3, but " - "x_dim.nbDims = %d, x_num_col_dims = %d.", - x_dim.nbDims, x_num_col_dims)); - auto output_name = op_desc.Output("Out").front(); - // add shuffle before fc - nvinfer1::Dims reshape_before_fc_dim; - // padding shape "x 1 x 1" - int padding_length = 3 - (x_dim.nbDims - x_num_col_dims); - reshape_before_fc_dim.nbDims = x_dim.nbDims + padding_length; - int cur_dim_index = reshape_before_fc_dim.nbDims - 1; - while (padding_length-- > 0) { - reshape_before_fc_dim.d[cur_dim_index--] = 1; - } - while (cur_dim_index >= 0) { - reshape_before_fc_dim.d[cur_dim_index--] = 0; - } - - auto* reshape_before_fc_layer = - TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *X); - reshape_before_fc_layer->setReshapeDimensions(reshape_before_fc_dim); - reshape_before_fc_layer->setName( - ("shuffle_before_fc(Output: " + output_name + ")").c_str()); - - // add fc layer - auto* fc_layer = TRT_ENGINE_ADD_LAYER( - engine_, FullyConnected, *reshape_before_fc_layer->getOutput(0), - n_output, weight.get(), bias.get()); - fc_layer->setName(("fc_layer(Output: " + output_name + ")").c_str()); - - // add shuffle after fc - nvinfer1::Dims reshape_after_fc_dim; - reshape_after_fc_dim.nbDims = x_num_col_dims + 1; - for (int i = 0; i < reshape_after_fc_dim.nbDims; i++) { - reshape_after_fc_dim.d[i] = 0; - } - - auto* reshape_after_fc_layer = - TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *fc_layer->getOutput(0)); - reshape_after_fc_layer->setReshapeDimensions(reshape_after_fc_dim); - - if (activation_type == "relu") { - reshape_after_fc_layer->setName( - ("shuffle_after_fc(Output: " + output_name + ")").c_str()); - nvinfer1::IActivationLayer* relu_layer = TRT_ENGINE_ADD_LAYER( - engine_, Activation, *(reshape_after_fc_layer->getOutput(0)), - nvinfer1::ActivationType::kRELU); - RreplenishLayerAndOutput(relu_layer, "relu_after_fc_shuffle", - {output_name}, test_mode); - } else { - RreplenishLayerAndOutput(reshape_after_fc_layer, "shuffle_after_fc", - {output_name}, test_mode); - } - return; + // Running the TRT Static Shape mode: x_num_col_dims-1 + if (!engine_->with_dynamic_shape()) { + x_num_col_dims--; } - // in order to handle situations in NLP models(input dims < 3, - // x_num_col_dims != 1, etc.), reshape input to perform FC correctly. - auto* reshape_itensor = X; - int input_dims = X->getDimensions().nbDims; - auto input_d = X->getDimensions().d; - int reshape_dim3[3] = {0}; - int reshape_dim4[4] = {0}; - PADDLE_ENFORCE_LE(x_num_col_dims, input_dims, - platform::errors::InvalidArgument( - "Params and input dims mismatch. Paddle-TRT FC " - "converter expects x_num_col_dims <= input dims")); - if (x_num_col_dims == 1) { - if (input_dims == 4) { - PADDLE_ENFORCE_EQ( - input_d[3], 1, - platform::errors::InvalidArgument( - "Invalid dimensions. When x_num_col_dims equals to 1 and input " - "dims equals to 4, the last dim of input must be 1, but got %d", - input_d[3])); - } - if (enable_int8) { - reshape_dim3[0] = 1; - for (int i = 0; i < 3; i++) { - reshape_dim3[0] *= input_d[i]; - if (i > 0) { - reshape_dim3[i] = 1; - } - } - } else { - for (int i = 0; i < 3; i++) { - if (i < input_dims) { - reshape_dim3[i] = input_d[i]; - } else { - reshape_dim3[i] = 1; - } - } - } - - nvinfer1::Dims3 reshape_dim(reshape_dim3[0], reshape_dim3[1], - reshape_dim3[2]); - auto* reshape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *X); - reshape_layer->setReshapeDimensions(reshape_dim); - reshape_itensor = reshape_layer->getOutput(0); - if (enable_int8) { - engine_->SetTensorDynamicRange(reshape_itensor, in_scale); - } - } else { - PADDLE_ENFORCE_NE(input_dims, 1, - platform::errors::InvalidArgument( - "Invalid dimensions. When x_num_col_dims equals to " - "2, input_dims should not be 1")); - - if (enable_int8) { - for (int i = 0; i < 4; i++) { - if (i == 0) { - reshape_dim4[i] = input_d[i]; - } else { - reshape_dim4[i] = 1; - if (i < input_dims) { - reshape_dim4[1] *= input_d[i]; - } - } - } + // If use tensorrt'oss, the x_dim and x_num_col_dims need change + if (engine_->use_oss() && engine_->with_ernie() && x_dim.nbDims == 4 && + x_dim.d[2] == 1 && x_dim.d[3] == 1 && x_num_col_dims == 2) { + x_num_col_dims = 1; + } + PADDLE_ENFORCE_GT( + x_dim.nbDims, x_num_col_dims, + platform::errors::InvalidArgument( + "Params and input dims mismatch. Paddle-TRT FC " + "converter expects x_dim.nbDims > x_num_col_dims, but " + "x_dim.nbDims : %d, x_num_col_dims : %d.", + x_dim.nbDims, x_num_col_dims)); + // add shuffle before fc + nvinfer1::Dims reshape_before_fc_dim; + reshape_before_fc_dim.nbDims = x_num_col_dims + 3; + // padding shape "* x q x 1 x 1" + for (int i = 0; i < reshape_before_fc_dim.nbDims; i++) { + reshape_before_fc_dim.d[i] = 1; + } + for (int i = 0; i < x_dim.nbDims; i++) { + if (i < x_num_col_dims) { + reshape_before_fc_dim.d[i] = 0; } else { - for (int i = 0; i < 4; i++) { - if (i < input_dims) { - reshape_dim4[i] = input_d[i]; - } else { - reshape_dim4[i] = 1; - } + if (x_dim.d[i] < 0) { + reshape_before_fc_dim.d[x_num_col_dims] = -1; + break; } + reshape_before_fc_dim.d[x_num_col_dims] *= x_dim.d[i]; } - nvinfer1::Dims4 reshape_dim(reshape_dim4[0], reshape_dim4[1], - reshape_dim4[2], reshape_dim4[3]); - auto* reshape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *X); - reshape_layer->setReshapeDimensions(reshape_dim); - reshape_itensor = reshape_layer->getOutput(0); - if (enable_int8) { - engine_->SetTensorDynamicRange(reshape_itensor, in_scale); - } + } + auto* reshape_before_fc_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *X); + reshape_before_fc_layer->setReshapeDimensions(reshape_before_fc_dim); + reshape_before_fc_layer->setName( + ("shuffle_before_fc(Output: " + output_name + ")").c_str()); + auto* reshape_itensor = reshape_before_fc_layer->getOutput(0); + if (enable_int8) { + engine_->SetTensorDynamicRange(reshape_itensor, in_scale); } regist_fc(reshape_itensor, n_output, weight, bias); } diff --git a/paddle/fluid/inference/tensorrt/convert/flatten_op.cc b/paddle/fluid/inference/tensorrt/convert/flatten_op.cc index 03a1c1672469e..322b42667fa30 100644 --- a/paddle/fluid/inference/tensorrt/convert/flatten_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flatten_op.cc @@ -25,7 +25,7 @@ namespace inference { namespace tensorrt { /* - * FlattenOp, only support static shape mode currently. + * FlattenOp trt converter */ class FlattenOpConverter : public OpConverter { public: @@ -35,21 +35,48 @@ class FlattenOpConverter : public OpConverter { // Declare inputs auto* input = engine_->GetITensor(op_desc.Input("X")[0]); int dims = input->getDimensions().nbDims; + nvinfer1::IShuffleLayer* layer = nullptr; + if (!engine_->with_dynamic_shape()) { + int dim_prod = 1; + for (int i = 0; i < dims; i++) { + int dim_i = input->getDimensions().d[i]; + PADDLE_ENFORCE_GT( + dim_i, 0, + platform::errors::InvalidArgument( + "flatten input dim should be > 0, but got %d.", dim_i)); + dim_prod *= dim_i; + } + nvinfer1::Dims flatten_dim; + flatten_dim.nbDims = 1; + flatten_dim.d[0] = dim_prod; + layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); + layer->setReshapeDimensions(flatten_dim); + } else { + auto* shape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shape, *input); + uint32_t reduce_dim = 1; - int dim_prod = 1; - for (int i = 0; i < dims; i++) { - int dim_i = input->getDimensions().d[i]; - PADDLE_ENFORCE_GT( - dim_i, 0, platform::errors::InvalidArgument( - "flatten input dim should be > 0, but got %d.", dim_i)); - dim_prod *= dim_i; + auto* reduce_prod_layer = TRT_ENGINE_ADD_LAYER( + engine_, Reduce, *(shape_layer->getOutput(0)), + nvinfer1::ReduceOperation::kPROD, reduce_dim, true); + int32_t* constant_weight_data = new int32_t[1]; + constant_weight_data[0] = -1; + TensorRTEngine::Weight constant_weight{ + nvinfer1::DataType::kINT32, static_cast(constant_weight_data), + 1}; + nvinfer1::Dims constant_dims; + constant_dims.nbDims = 1; + constant_dims.d[0] = 1; + auto* constant_layer = TRT_ENGINE_ADD_LAYER( + engine_, Constant, constant_dims, constant_weight.get()); + std::vector itensors; + itensors.push_back(constant_layer->getOutput(0)); + itensors.push_back(reduce_prod_layer->getOutput(0)); + auto* concat_layer = + TRT_ENGINE_ADD_LAYER(engine_, Concatenation, itensors.data(), 2); + concat_layer->setAxis(0); + layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); + layer->setInput(1, *(concat_layer->getOutput(0))); } - nvinfer1::Dims flatten_dim; - flatten_dim.nbDims = 1; - flatten_dim.d[0] = dim_prod; - auto* layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); - layer->setReshapeDimensions(flatten_dim); - auto output_name = op_desc.Output("Out")[0]; RreplenishLayerAndOutput(layer, "flatten", {output_name}, test_mode); } diff --git a/paddle/fluid/inference/tensorrt/convert/reshape_op.cc b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc index 3d8c72728c667..489603e20cda2 100644 --- a/paddle/fluid/inference/tensorrt/convert/reshape_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/reshape_op.cc @@ -34,7 +34,7 @@ class ReshapeOpConverter : public OpConverter { framework::OpDesc op_desc(op, nullptr); // Declare inputs auto* input = engine_->GetITensor(op_desc.Input("X")[0]); - const std::vector& shape = + std::vector shape = BOOST_GET_CONST(std::vector, op_desc.GetAttr("shape")); int nbDims_num = shape.size(); nvinfer1::Dims reshape_dim; diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 44611d1d5959d..59b196e3d92be 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -300,23 +300,14 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, if (axis.size() >= nvinfer1::Dims::MAX_DIMS) return false; } } - if (op_type == "flatten2") { - // flatten doesn't support dynamic shape currently - if (!desc.HasAttr("axis")) { - return false; - } else { - if (with_dynamic_shape) return false; - int axis = BOOST_GET_CONST(int, desc.GetAttr("axis")); - if (axis != 1) return false; - } - } - - if (op_type == "flatten") { - // flatten doesn't support dynamic shape currently + if (op_type == "flatten2" || op_type == "flatten") { if (!desc.HasAttr("axis")) { return false; } else { +#if IS_TRT_VERSION_GE(7130) +#else if (with_dynamic_shape) return false; +#endif int axis = BOOST_GET_CONST(int, desc.GetAttr("axis")); if (axis != 1) return false; } @@ -685,20 +676,19 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } } - if (op_type == "reduce_sum") { - if (!with_dynamic_shape) { - VLOG(3) << "the reduce_sum does not support static shape yet"; - return false; - } - - if (!(desc.HasAttr("keep_dim") && desc.HasAttr("dim") && - desc.HasAttr("reduce_all"))) { - VLOG(3) << "the reduce_sum does not have attr (keep_dim or dim or " - "reduce_all)"; + if (op_type == "fc") { + int x_num_col_dims = + desc.HasAttr("x_num_col_dims") + ? BOOST_GET_CONST(int, desc.GetAttr("x_num_col_dims")) + : (desc.HasAttr("in_num_col_dims") + ? BOOST_GET_CONST(int, desc.GetAttr("in_num_col_dims")) + : 1); + if (x_num_col_dims < 1) { + VLOG(3) << "converter expects x_num_col_dims >= 1, " + "but x_num_col_dims = %d."; return false; } } - if (op_type == "reshape" || op_type == "reshape2") { if (!desc.HasAttr("shape")) { return false; @@ -712,6 +702,21 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, if (shape.size() >= nvinfer1::Dims::MAX_DIMS) return false; } } + + if (op_type == "reduce_sum") { + if (!with_dynamic_shape) { + VLOG(3) << "the reduce_sum does not support static shape yet"; + return false; + } + + if (!(desc.HasAttr("keep_dim") && desc.HasAttr("dim") && + desc.HasAttr("reduce_all"))) { + VLOG(3) << "the reduce_sum does not have attr (keep_dim or dim or " + "reduce_all)"; + return false; + } + } + if ((*teller)(op_type, desc, use_no_calib_int8)) return true; } return false; From 18043ab5b478d5ab665e395338df63f9a888e725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=AD=A3?= <2042519524@qq.com> Date: Mon, 21 Jun 2021 11:19:13 +0800 Subject: [PATCH 127/156] fix the but that concat op can't support uint8 (#33667) --- paddle/fluid/operators/concat_op.cc | 2 +- paddle/fluid/operators/concat_op.cu.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/operators/concat_op.cc b/paddle/fluid/operators/concat_op.cc index 68a52a79e4ce3..6095516f92fa5 100644 --- a/paddle/fluid/operators/concat_op.cc +++ b/paddle/fluid/operators/concat_op.cc @@ -244,4 +244,4 @@ REGISTER_OP_CPU_KERNEL( ops::ConcatGradKernel, ops::ConcatGradKernel, - ops::ConcatKernel); + ops::ConcatGradKernel); diff --git a/paddle/fluid/operators/concat_op.cu.cc b/paddle/fluid/operators/concat_op.cu.cc index 8732556acb9fd..63025c3bd030f 100644 --- a/paddle/fluid/operators/concat_op.cu.cc +++ b/paddle/fluid/operators/concat_op.cu.cc @@ -33,4 +33,4 @@ REGISTER_OP_CUDA_KERNEL( ops::ConcatGradKernel, ops::ConcatGradKernel, ops::ConcatGradKernel, - ops::ConcatKernel); + ops::ConcatGradKernel); From cdeffff4fcc2ab1965b040582753d01ebfee05b9 Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Mon, 21 Jun 2021 15:32:12 +0800 Subject: [PATCH 128/156] fix gpt2 train loss Nan problem by add a line __syncthreads in BlockReduceSum (#33659) --- paddle/fluid/operators/correlation_op.cu | 1 + paddle/fluid/operators/layer_norm_op.cu | 17 ++++++++++------- paddle/fluid/operators/math/math_cuda_utils.h | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/operators/correlation_op.cu b/paddle/fluid/operators/correlation_op.cu index a51fce8132418..76e10f90ef833 100644 --- a/paddle/fluid/operators/correlation_op.cu +++ b/paddle/fluid/operators/correlation_op.cu @@ -42,6 +42,7 @@ __forceinline__ __device__ T blockReduceSum(T val) { int wid = threadIdx.x / warpSize; val = warpReduceSum(val); + __syncthreads(); if (lane == 0) shared[wid] = val; __syncthreads(); diff --git a/paddle/fluid/operators/layer_norm_op.cu b/paddle/fluid/operators/layer_norm_op.cu index f955011675cf5..25c722358c4e3 100755 --- a/paddle/fluid/operators/layer_norm_op.cu +++ b/paddle/fluid/operators/layer_norm_op.cu @@ -64,17 +64,16 @@ static __forceinline__ __device__ U WarpReduceSum(U val) { } template -__forceinline__ __device__ U BlockReduceSum(U val) { - static __shared__ U shared[32]; +__forceinline__ __device__ U BlockReduceSum(U val, U *shared) { int lane = threadIdx.x % warpSize; int wid = threadIdx.x / warpSize; val = WarpReduceSum(val); // Each warp performs partial reduction + __syncthreads(); if (lane == 0) shared[wid] = val; // Write reduced value to shared memory __syncthreads(); // Wait for all partial reductions - // read from shared memory only if that warp existed val = (threadIdx.x < blockDim.x / warpSize) ? shared[lane] : static_cast(0); @@ -183,6 +182,8 @@ __global__ void LayerNormForward(const T *x, const U *scale, const U *bias, int64_t feature_size) { __shared__ U mean_share; __shared__ U var_share; + __shared__ U shared_mean[32]; + __shared__ U shared_var[32]; int64_t beg_idx = blockIdx.x * feature_size + threadIdx.x; int64_t end_idx = (blockIdx.x + 1) * feature_size; @@ -196,8 +197,8 @@ __global__ void LayerNormForward(const T *x, const U *scale, const U *bias, var_val += (tmp * tmp); } - mean_val = BlockReduceSum(mean_val); - var_val = BlockReduceSum(var_val); + mean_val = BlockReduceSum(mean_val, shared_mean); + var_val = BlockReduceSum(var_val, shared_var); if (threadIdx.x == 0) { auto scale = static_cast(1.) / static_cast(feature_size); @@ -541,8 +542,10 @@ __global__ void LayerNormBackwardGradientAll( } } - d_scale_partial = BlockReduceSum(d_scale_partial); - d_bias_partial = BlockReduceSum(d_bias_partial); + __shared__ U shared_scale[32]; + __shared__ U shared_bias[32]; + d_scale_partial = BlockReduceSum(d_scale_partial, shared_scale); + d_bias_partial = BlockReduceSum(d_bias_partial, shared_bias); if (threadIdx.x == 0) { d_scale[blockIdx.x + col_offset] = d_scale_partial; diff --git a/paddle/fluid/operators/math/math_cuda_utils.h b/paddle/fluid/operators/math/math_cuda_utils.h index e97dbd20ca142..8de4e8221c0e4 100644 --- a/paddle/fluid/operators/math/math_cuda_utils.h +++ b/paddle/fluid/operators/math/math_cuda_utils.h @@ -188,6 +188,7 @@ __inline__ __device__ T blockReduceSum(T val, unsigned mask) { val = warpReduceSum(val, mask); + __syncthreads(); if (lane == 0) shared[wid] = val; __syncthreads(); From bf3161bdb129922cd6fee75630983d5ed89f9895 Mon Sep 17 00:00:00 2001 From: Pei Yang Date: Tue, 22 Jun 2021 10:41:34 +0800 Subject: [PATCH 129/156] fix emb_eltwise_ln gpu_id bug (#33701) (#33706) --- paddle/fluid/inference/api/analysis_config.cc | 1 - paddle/fluid/inference/api/analysis_predictor.cc | 4 ++-- paddle/fluid/inference/api/paddle_analysis_config.h | 2 +- .../tensorrt/plugin/emb_eltwise_layernorm_plugin.cu | 2 +- .../inference/tests/api/trt_dynamic_shape_ernie_test.cc | 5 ----- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 853c1ac1da874..b5ca0ef592439 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -421,7 +421,6 @@ void AnalysisConfig::Update() { pass_builder()->AppendPass(pass); } } - LOG(INFO) << "use_dlnne_:" << use_dlnne_ << std::endl; if (use_dlnne_) { pass_builder()->ClearPasses(); for (const auto &pass : kDlnneSubgraphPasses) { diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 215174c12ce3b..b205d553c99ca 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -152,8 +152,8 @@ bool AnalysisPredictor::Init( : platform::ProfilerState::kCPU; platform::EnableProfiler(tracking_device); } else { - LOG(INFO) << "Profiler is deactivated, and no profiling report will be " - "generated."; + VLOG(2) << "Profiler is deactivated, and no profiling report will be " + "generated."; } // no matter with or without MKLDNN diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 2bbd4bb837a22..f9e4869934a0f 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -294,7 +294,7 @@ struct PD_INFER_DECL AnalysisConfig { /// workspace. /// \param max_batch_size The maximum batch size of this prediction task, /// better set as small as possible for less performance loss. - /// \param min_subgrpah_size The minimum TensorRT subgraph size needed, if a + /// \param min_subgraph_size The minimum TensorRT subgraph size needed, if a /// subgraph is smaller than this, it will not be transferred to TensorRT /// engine. /// \param precision The precision used in TensorRT. diff --git a/paddle/fluid/inference/tensorrt/plugin/emb_eltwise_layernorm_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/emb_eltwise_layernorm_plugin.cu index 6d3872aaeb8a7..c873b1fc310de 100644 --- a/paddle/fluid/inference/tensorrt/plugin/emb_eltwise_layernorm_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/emb_eltwise_layernorm_plugin.cu @@ -134,7 +134,7 @@ int EmbEltwiseLayernormPluginDynamicImpl::enqueue( int batch = id_dims.d[0]; int seq_len = id_dims.d[1]; int input_num = embs_.size(); - + cudaGetDevice(&device_id_); auto in_ptr_gpu_d = in_ptr_tensor_.mutable_data(platform::CUDAPlace(device_id_)); auto emb_ptr_gpu_d = diff --git a/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc b/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc index 45dff9f4c3710..a45b78f05e73c 100644 --- a/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc +++ b/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc @@ -29,11 +29,6 @@ void run(const AnalysisConfig& config, std::vector* out_data) { int run_batch = 1; const int run_seq_len = 128; - std::vector tmp_input; - std::vector tmp_four_input; - tmp_input.reserve(run_batch * run_seq_len); - tmp_four_input.reserve(run_batch * run_seq_len); - int64_t i0[run_seq_len] = { 1, 3558, 4, 75, 491, 89, 340, 313, 93, 4, 255, 10, 75, 321, 4095, 1902, 4, 134, 49, 75, 311, 14, 44, 178, 543, 15, 12043, 2, From 3b3bd932c7e9dd65f40fdc3b2d9f0f8a032e2413 Mon Sep 17 00:00:00 2001 From: ceci3 Date: Tue, 22 Jun 2021 11:53:00 +0800 Subject: [PATCH 130/156] add layernorm (#33610) (#33707) --- .../paddle/fluid/contrib/slim/quantization/quantization_pass.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py b/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py index ec215a3e5757e..320c14d4e9ca4 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py @@ -81,6 +81,7 @@ "transpose", "pad2d", "reshape", + "layer_norm", ] # list op real input and output names, to avoid processing input such as AxisTensor. From a029d36e342e9915667d2decf660664363259d4c Mon Sep 17 00:00:00 2001 From: jiangcheng Date: Tue, 22 Jun 2021 14:48:48 +0800 Subject: [PATCH 131/156] [Cherry-pick] solve ANSI escape sequences print error in cmd and powershell (#33689) (#33715) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解决windows cmd和powershell显示乱码的问题 --- python/paddle/fluid/dygraph/varbase_patch_methods.py | 4 ++++ python/paddle/utils/deprecated.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index 644e25ab9183b..17cd499bfee5f 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -16,6 +16,7 @@ import numpy as np import warnings import weakref +import sys import paddle from .. import framework @@ -372,6 +373,9 @@ def grad(self): """ msg = "tensor.grad will return the tensor value of the gradient." warning_msg = "\033[93m\nWarning:\n%s \033[0m" % (msg) + # ensure ANSI escape sequences print correctly in cmd and powershell + if sys.platform.lower() == 'win32': + warning_msg = "\nWarning:\n%s " % (msg) warnings.warn(warning_msg) return self._grad_ivar() diff --git a/python/paddle/utils/deprecated.py b/python/paddle/utils/deprecated.py index e3839d9767d21..b17bd70c91af2 100755 --- a/python/paddle/utils/deprecated.py +++ b/python/paddle/utils/deprecated.py @@ -18,6 +18,7 @@ import warnings import functools import paddle +import sys __all__ = [] @@ -99,6 +100,10 @@ def wrapper(*args, **kwargs): func.__module__, func.__name__)) warningmsg = "\033[93m\nWarning:\n%s \033[0m" % (msg) + # ensure ANSI escape sequences print correctly in cmd and powershell + if sys.platform.lower() == 'win32': + warningmsg = "\nWarning:\n%s " % (msg) + v_current = [int(i) for i in paddle.__version__.split(".")] v_current += [0] * (4 - len(v_current)) v_since = [int(i) for i in _since.split(".")] From 1e62c239d323354eccfc974d4e2e6496f93d848e Mon Sep 17 00:00:00 2001 From: Roc <30228238+sljlp@users.noreply.github.com> Date: Tue, 22 Jun 2021 16:09:04 +0800 Subject: [PATCH 132/156] Dynamic amp support sync_batch_norm op (#32770) (#33709) --- paddle/fluid/imperative/amp_auto_cast.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc index fd2bb6e5c9952..b4154737e0fbc 100644 --- a/paddle/fluid/imperative/amp_auto_cast.cc +++ b/paddle/fluid/imperative/amp_auto_cast.cc @@ -160,7 +160,8 @@ NameVarBaseMap AutoCastInputs(const std::string& op_type, if (AmpOperators::Instance().GetMutableAllowOps()->count(op_type)) { for (auto& pair : new_ins) { // NOTE(zhiqiu): batch_norm and layer_norm support only input x is fp16. - if ((op_type == "batch_norm" || op_type == "layer_norm") && + if ((op_type == "batch_norm" || op_type == "layer_norm" || + op_type == "sync_batch_norm") && pair.first != "X") { continue; } @@ -191,7 +192,8 @@ NameVarBaseMap AutoCastInputs(const std::string& op_type, } for (auto& pair : new_ins) { // NOTE(zhiqiu): batch_norm and layer_norm support only input x is fp16. - if ((op_type == "batch_norm" || op_type == "layer_norm") && + if ((op_type == "batch_norm" || op_type == "layer_norm" || + op_type == "sync_batch_norm") && pair.first == "X" && dst_type == framework::proto::VarType::FP32) { continue; } From 89fdd6c8f569fe20b7699ca20488f603ca2636ba Mon Sep 17 00:00:00 2001 From: wenbin Date: Mon, 28 Jun 2021 20:49:30 +0800 Subject: [PATCH 133/156] Fix wrong scale length for QkvToContext (#33763) (#33784) --- .../tensorrt/plugin/qkv_to_context_plugin.cu | 2 +- .../tests/api/trt_dynamic_shape_ernie_test.cc | 62 +++++++++++++------ 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu index 214e1a81e7dc0..5f10e5821c4f7 100644 --- a/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/qkv_to_context_plugin.cu @@ -299,7 +299,7 @@ int QkvToContextPluginDynamic::enqueue( platform::DeviceContextPool::Instance().Get( platform::CUDAPlace(device_id))); - int n_q = seq_len * head_number_ * head_size_; + int n_q = seq_len * head_number_ * head_size_ * batch; constexpr int threads = 128; int blocks = (n_q + threads - 1) / threads; diff --git a/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc b/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc index a45b78f05e73c..e449fb5096e6e 100644 --- a/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc +++ b/paddle/fluid/inference/tests/api/trt_dynamic_shape_ernie_test.cc @@ -22,51 +22,60 @@ limitations under the License. */ namespace paddle { namespace inference { -void run(const AnalysisConfig& config, std::vector* out_data) { +void run(const AnalysisConfig& config, std::vector* out_data, int bs) { auto predictor = CreatePaddlePredictor(config); auto input_names = predictor->GetInputNames(); - int run_batch = 1; + int run_batch = bs; const int run_seq_len = 128; + size_t len = run_batch * run_seq_len; - int64_t i0[run_seq_len] = { + int64_t i0_bs1[run_seq_len] = { 1, 3558, 4, 75, 491, 89, 340, 313, 93, 4, 255, 10, 75, 321, 4095, 1902, 4, 134, 49, 75, 311, 14, 44, 178, 543, 15, 12043, 2, 75, 201, 340, 9, 14, 44, 486, 218, 1140, 279, 12043, 2}; - int64_t i1[run_seq_len] = { + int64_t i1_bs1[run_seq_len] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - int64_t i2[run_seq_len] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}; - float i3[run_seq_len] = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; - + int64_t i2_bs1[run_seq_len] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}; + float i3_bs1[run_seq_len] = { + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; + std::vector i0_data(len), i1_data(len), i2_data(len); + std::vector i3_data(len); + + for (size_t i = 0; i < len; i++) { + i0_data[i] = i0_bs1[i % run_seq_len]; + i1_data[i] = i1_bs1[i % run_seq_len]; + i2_data[i] = i2_bs1[i % run_seq_len]; + i3_data[i] = i3_bs1[i % run_seq_len]; + } // first input auto input_t = predictor->GetInputTensor(input_names[0]); input_t->Reshape({run_batch, run_seq_len, 1}); - input_t->copy_from_cpu(i0); + input_t->copy_from_cpu(i0_data.data()); // second input auto input_t2 = predictor->GetInputTensor(input_names[1]); input_t2->Reshape({run_batch, run_seq_len, 1}); - input_t2->copy_from_cpu(i1); + input_t2->copy_from_cpu(i1_data.data()); // third input. auto input_t3 = predictor->GetInputTensor(input_names[2]); input_t3->Reshape({run_batch, run_seq_len, 1}); - input_t3->copy_from_cpu(i2); + input_t3->copy_from_cpu(i2_data.data()); auto input_t4 = predictor->GetInputTensor(input_names[3]); input_t4->Reshape({run_batch, run_seq_len, 1}); - input_t4->copy_from_cpu(i3); + input_t4->copy_from_cpu(i3_data.data()); ASSERT_TRUE(predictor->ZeroCopyRun()); @@ -79,8 +88,8 @@ void run(const AnalysisConfig& config, std::vector* out_data) { output_t->copy_to_cpu(out_data->data()); } -void trt_ernie(bool with_fp16, std::vector result, - float near_tolerance) { +void trt_ernie(bool with_fp16, std::vector result, float near_tolerance, + int batch_size = 1) { AnalysisConfig config; std::string model_dir = FLAGS_infer_model; SetConfig(&config, model_dir, true); @@ -120,7 +129,7 @@ void trt_ernie(bool with_fp16, std::vector result, config.SetTRTDynamicShapeInfo(min_input_shape, max_input_shape, opt_input_shape); std::vector out_data; - run(config, &out_data); + run(config, &out_data, batch_size); for (size_t i = 0; i < out_data.size(); i++) { EXPECT_NEAR(result[i], out_data[i], near_tolerance); @@ -139,6 +148,19 @@ TEST(AnalysisPredictor, fp16) { #endif } +TEST(AnalysisPredictor, no_fp16_bs2) { + std::vector result = {0.597841, 0.219972, 0.182187, + 0.597841, 0.219972, 0.182187}; + trt_ernie(false, result, 1e-5, 2); +} + +TEST(AnalysisPredictor, fp16_bs2) { +#ifdef TRT_PLUGIN_FP16_AVALIABLE + std::vector result = {0.598, 0.219, 0.182, 0.598, 0.219, 0.182}; + trt_ernie(true, result, 4e-3, 2); +#endif +} + // ernie_varlen std::shared_ptr InitPredictor() { paddle_infer::Config config; From 3749af59f322f703d395547e04bd78c01e81bd34 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Thu, 1 Jul 2021 10:57:34 +0800 Subject: [PATCH 134/156] [Dy2stat]Specify gast version in requirements.txt (#33850) (#33865) cherry-pick Specify gast version in requirements.txt --- python/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index 609a4b34e8f1a..14bd5e7caa6f5 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,7 +3,7 @@ numpy>=1.13, <=1.16.4 ; python_version<"3.5" numpy>=1.13 ; python_version>="3.5" and platform_system != "Windows" numpy>=1.13, <=1.19.3 ; python_version>="3.5" and platform_system == "Windows" protobuf>=3.1.0 -gast>=0.3.3 ; platform_system != "Windows" +gast>=0.3.3, <=0.4.0 ; platform_system != "Windows" gast==0.3.3 ; platform_system == "Windows" Pillow six From 702610efc58bcaeab3c88c0b222e21064581fc97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=98=8E=E5=86=AC?= <78149749+winter-wang@users.noreply.github.com> Date: Thu, 1 Jul 2021 13:41:46 +0800 Subject: [PATCH 135/156] fix the opt path create error in windows, test=develop (#33853) (#33885) --- paddle/fluid/inference/analysis/helper.h | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/inference/analysis/helper.h b/paddle/fluid/inference/analysis/helper.h index cace420d87c9d..ebea4d0386090 100644 --- a/paddle/fluid/inference/analysis/helper.h +++ b/paddle/fluid/inference/analysis/helper.h @@ -182,15 +182,16 @@ static bool PathExists(const std::string &path) { } static std::string GetDirRoot(const std::string &path) { - char sep = '/'; - -#ifdef _WIN32 - sep = '\\'; -#endif - - size_t i = path.rfind(sep, path.length()); - if (i != std::string::npos) { - return (path.substr(0, i)); + char sep_1 = '/', sep_2 = '\\'; + + size_t i_1 = path.rfind(sep_1, path.length()); + size_t i_2 = path.rfind(sep_2, path.length()); + if (i_1 != std::string::npos && i_2 != std::string::npos) { + return path.substr(0, std::max(i_1, i_2)); + } else if (i_1 != std::string::npos) { + return path.substr(0, i_1); + } else if (i_2 != std::string::npos) { + return path.substr(0, i_2); } return path; } From bedcf0dd98e30accd32969a70d5729ef8a8d2f15 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 1 Jul 2021 19:09:04 +0800 Subject: [PATCH 136/156] [cherry-pick] fix bug when the cuda kernel config exceeds dims max (#33748) (#33893) fix bug when the cuda kernel config exceeds dims max --- paddle/fluid/operators/layer_norm_op.cu | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) mode change 100755 => 100644 paddle/fluid/operators/layer_norm_op.cu diff --git a/paddle/fluid/operators/layer_norm_op.cu b/paddle/fluid/operators/layer_norm_op.cu old mode 100755 new mode 100644 index 25c722358c4e3..0410e05115860 --- a/paddle/fluid/operators/layer_norm_op.cu +++ b/paddle/fluid/operators/layer_norm_op.cu @@ -399,9 +399,9 @@ __global__ void LayerNormBackwardComputeGradInput( const U *__restrict__ mean, const U *__restrict__ var, const float epsilon, const U *gamma, T *grad_input) { #ifdef __HIPCC__ - for (auto i1 = hipBlockIdx_y; i1 < n1; i1 += hipGridDim_y) { + for (auto i1 = hipBlockIdx_x; i1 < n1; i1 += hipGridDim_x) { #else - for (auto i1 = blockIdx.y; i1 < n1; i1 += gridDim.y) { + for (auto i1 = blockIdx.x; i1 < n1; i1 += gridDim.x) { #endif U sum_loss1 = U(0); U sum_loss2 = U(0); @@ -867,9 +867,8 @@ static void LayerNormBackward(const T *x, const T *d_y, const U *scale, constexpr int BDIMX1 = 32; constexpr int BDIMY1 = 4; dim3 threads1(BDIMX1, BDIMY1, 1); - const dim3 blocks1(1, batch_size, 1); LayerNormBackwardComputeGradInput< - T, U, BDIMX1, BDIMY1><<>>( + T, U, BDIMX1, BDIMY1><<>>( d_y, x, batch_size, feature_size, mean, var, epsilon, scale, d_x); break; } From aa12737b99733eb2da27417f9a3288b945b7af99 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Fri, 2 Jul 2021 10:08:03 +0800 Subject: [PATCH 137/156] [Dy2Stat]Support Python3 type hint (#33745) (#33914) [Dy2Stat] Support Python3 type hint (#33745) --- .../fluid/dygraph/dygraph_to_static/utils.py | 11 +- .../dygraph_to_static/test_origin_info.py | 8 +- .../dygraph_to_static/test_typing.py | 124 ++++++++++++++++++ 3 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py index 001116a74c9cc..1513b9f5222e6 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py @@ -487,8 +487,7 @@ def remove_if_exit(filepath): os.remove(filepath) source = ast_to_source_code(ast_root) - import_fluid = "import paddle\nimport paddle.fluid as fluid\n" - source = import_fluid + source + source = _inject_import_statements() + source if six.PY2: source = source.encode('utf-8') @@ -528,6 +527,14 @@ def remove_if_exit(filepath): return callable_func, f.name +def _inject_import_statements(): + import_statements = [ + "import paddle", "import paddle.fluid as fluid", "from typing import *", + "import numpy as np" + ] + return '\n'.join(import_statements) + '\n' + + def recover_globals_attribute(src_obj, dst_obj): attr_name = '__globals__' diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py index 144b16873aa9b..016a1b3b588ab 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py @@ -65,7 +65,7 @@ def set_test_func(self): self.func = simple_func def set_static_lineno(self): - self.static_abs_lineno_list = [3, 4, 5] + self.static_abs_lineno_list = [5, 6, 7] def set_dygraph_info(self): self.line_num = 3 @@ -149,7 +149,7 @@ def set_test_func(self): self.func = nested_func def set_static_lineno(self): - self.static_abs_lineno_list = [3, 5, 6, 7, 8] + self.static_abs_lineno_list = [5, 7, 8, 9, 10] def set_dygraph_info(self): self.line_num = 5 @@ -174,7 +174,7 @@ def set_test_func(self): self.func = decorated_func def set_static_lineno(self): - self.static_abs_lineno_list = [3, 4] + self.static_abs_lineno_list = [5, 6] def set_dygraph_info(self): self.line_num = 2 @@ -208,7 +208,7 @@ def set_test_func(self): self.func = decorated_func2 def set_static_lineno(self): - self.static_abs_lineno_list = [3, 4] + self.static_abs_lineno_list = [5, 6] def set_dygraph_info(self): self.line_num = 2 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py new file mode 100644 index 0000000000000..c3c0453bde3f4 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py @@ -0,0 +1,124 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import unittest +import numpy as np +from typing import Tuple, List, Dict, TypeVar + + +class BaseLayer(paddle.nn.Layer): + def __init__(self, in_size, out_size): + super(BaseLayer, self).__init__() + self._linear = paddle.nn.Linear(in_size, out_size) + + def build(self, x): + out1 = self._linear(x) + out2 = paddle.mean(out1) + return out1, out2 + + +class LinearNetWithTuple(BaseLayer): + def __init__(self, in_size, out_size): + super(LinearNetWithTuple, self).__init__(in_size, out_size) + + def forward(self, x) -> Tuple[paddle.Tensor, str]: + out1, out2 = self.build(x) + return (out2, 'str') + + +class LinearNetWithTuple2(BaseLayer): + def __init__(self, in_size, out_size): + super(LinearNetWithTuple2, self).__init__(in_size, out_size) + + def forward(self, x) -> Tuple[paddle.Tensor, np.array]: + out1, out2 = self.build(x) + return (out2, np.ones([4, 16])) + + +class LinearNetWithList(BaseLayer): + def __init__(self, in_size, out_size): + super(LinearNetWithList, self).__init__(in_size, out_size) + + def forward(self, x) -> List[paddle.Tensor]: + out1, out2 = self.build(x) + return [out2] + + +class LinearNetWithDict(BaseLayer): + def __init__(self, in_size, out_size): + super(LinearNetWithDict, self).__init__(in_size, out_size) + + def forward(self, x) -> Dict[str, paddle.Tensor]: + out1, out2 = self.build(x) + return {'out': out2} + + +class TestTyping(unittest.TestCase): + def setUp(self): + self.in_num = 16 + self.out_num = 16 + self.x = paddle.randn([4, 16]) + self.spec = [paddle.static.InputSpec(shape=[None, 16], dtype='float32')] + + def build_net(self): + return LinearNetWithTuple(self.in_num, self.out_num) + + def save_and_load(self, suffix=''): + path = './layer_typing_' + suffix + paddle.jit.save(self.net, path, input_spec=self.spec) + return paddle.jit.load(path) + + def run_dy(self): + out, _ = self.net(self.x) + return out + + def test_type(self): + self.net = self.build_net() + out = self.run_dy() + load_net = self.save_and_load('tuple') + load_out = load_net(self.x) + self.assertTrue(np.allclose(out, load_out)) + + +class TestTypingTuple(TestTyping): + def build_net(self): + return LinearNetWithTuple2(self.in_num, self.out_num) + + def run_dy(self): + out, np_data = self.net(self.x) + self.assertTrue(np.equal(np_data, np.ones_like(np_data)).all()) + return out + + +class TestTypingList(TestTyping): + def build_net(self): + return LinearNetWithList(self.in_num, self.out_num) + + def run_dy(self): + out = self.net(self.x)[0] + return out + + +class TestTypingDict(TestTyping): + def build_net(self): + return LinearNetWithDict(self.in_num, self.out_num) + + def run_dy(self): + out = self.net(self.x)['out'] + return out + + +if __name__ == '__main__': + unittest.main() From adca05f21580fceb09a524adac29ee725d436649 Mon Sep 17 00:00:00 2001 From: Zhou Wei <1183042833@qq.com> Date: Fri, 2 Jul 2021 11:36:27 +0800 Subject: [PATCH 138/156] [cherry-pick2.1]polish avx/no_avx install error message (#33818) (#33905) cherry-pick #33818 --- python/paddle/fluid/core.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/python/paddle/fluid/core.py b/python/paddle/fluid/core.py index 9e931ad40c57a..dae1e0cf296a2 100644 --- a/python/paddle/fluid/core.py +++ b/python/paddle/fluid/core.py @@ -298,17 +298,13 @@ def to_list(s): else: from .. import compat as cpt sys.stderr.write( - "WARNING: AVX is supported on local machine, but you have installed " - "paddlepaddle without avx core. Hence, no_avx core which has worse " - "preformance will be imported.\nYou could reinstall paddlepaddle by " - "'python -m pip install --force-reinstall paddlepaddle-gpu[==version]' or rebuild " - "paddlepaddle WITH_AVX=ON to get better performance.\n" - "The original error is: %s\n" % cpt.get_exception_message(e)) + "Hint: Your machine support AVX, but the installed paddlepaddle doesn't have avx core. " + "Hence, no-avx core with worse preformance will be imported.\nIf you like, you could " + "reinstall paddlepaddle by 'python -m pip install --force-reinstall paddlepaddle-gpu[==version]' " + "to get better performance.\nThe original error is: %s\n" % + cpt.get_exception_message(e)) load_noavx = True else: - sys.stderr.write( - "WARNING: AVX is not support on your machine. Hence, no_avx core will be imported, " - "It has much worse preformance than avx core.\n") load_noavx = True if load_noavx: @@ -355,17 +351,14 @@ def to_list(s): current_path + os.sep + 'core_noavx.' + core_suffix + '\n') elif avx_supported(): sys.stderr.write( - "Error: AVX is support on your machine, but you have installed " - "paddlepaddle without avx core, you should reinstall paddlepaddle by " - "'python -m pip install --force-reinstall paddlepaddle-gpu[==version]\n" + "Error: The installed PaddlePaddle is incorrect. You should reinstall it by " + "'python -m pip install --force-reinstall paddlepaddle-gpu[==version]'\n" ) else: sys.stderr.write( - "Error: AVX is not support on your machine, but you have installed " - "paddlepaddle without no_avx core, you should reinstall paddlepaddle by " - "'python -m pip install --force-reinstall paddlepaddle-gpu[==version] -f " - "https://paddlepaddle.org.cn/whl/mkl/stable/noavx.html or " - "https://paddlepaddle.org.cn/whl/openblas/stable/noavx.html\n") + "Error: Your machine doesn't support AVX, but the installed PaddlePaddle is avx core, " + "you should reinstall paddlepaddle with no-avx core.\n") + raise e From 50cb94575091c718acbf2efbec6f431f4f212c5d Mon Sep 17 00:00:00 2001 From: TCChenLong <1300851984@qq.com> Date: Thu, 1 Jul 2021 20:00:27 +0800 Subject: [PATCH 139/156] update readme test=document_fix --- README.md | 4 ++-- README_cn.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e8a7013d0b443..89bffbafd9ebd 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ PaddlePaddle is originated from industrial practices with dedication and commitm ## Installation -### Latest PaddlePaddle Release: [v2.0](https://github.com/PaddlePaddle/Paddle/tree/release/2.0) +### Latest PaddlePaddle Release: [v2.1](https://github.com/PaddlePaddle/Paddle/tree/release/2.1) Our vision is to enable deep learning for everyone via PaddlePaddle. Please refer to our [release announcement](https://github.com/PaddlePaddle/Paddle/releases) to track the latest features of PaddlePaddle. @@ -36,7 +36,7 @@ pip install paddlepaddle-gpu ``` More infomation about installation, please view [Quick Install](https://www.paddlepaddle.org.cn/install/quick) -Now our developers can acquire Tesla V100 online computing resources for free. If you create a program by AI Studio, you will obtain 10 hours to train models online per day. [Click here to start](https://aistudio.baidu.com/aistudio/index). +Now our developers can acquire Tesla V100 online computing resources for free. If you create a program by AI Studio, you will obtain 8 hours to train models online per day. [Click here to start](https://aistudio.baidu.com/aistudio/index). ## FOUR LEADING TECHNOLOGIES diff --git a/README_cn.md b/README_cn.md index 7a10cba284549..72ecadd379487 100644 --- a/README_cn.md +++ b/README_cn.md @@ -19,7 +19,7 @@ ## 安装 -### PaddlePaddle最新版本: [v2.0](https://github.com/PaddlePaddle/Paddle/tree/release/2.0) +### PaddlePaddle最新版本: [v2.1](https://github.com/PaddlePaddle/Paddle/tree/release/2.1) 跟进PaddlePaddle最新特性请参考我们的[版本说明](https://github.com/PaddlePaddle/Paddle/releases) @@ -32,7 +32,7 @@ pip install paddlepaddle-gpu ``` 更多安装信息详见官网 [安装说明](https://www.paddlepaddle.org.cn/install/quick) -PaddlePaddle用户可领取**免费Tesla V100在线算力资源**，训练模型更高效。**每日登陆即送10小时**，[前往使用免费算力](https://aistudio.baidu.com/aistudio/index)。 +PaddlePaddle用户可领取**免费Tesla V100在线算力资源**，训练模型更高效。**每日登陆即送8小时**，[前往使用免费算力](https://aistudio.baidu.com/aistudio/index)。 ## 四大领先技术 From 16ed3cc976b5e7a2621628d9e2e92cab493db6d8 Mon Sep 17 00:00:00 2001 From: iducn <45056973+iducn@users.noreply.github.com> Date: Fri, 2 Jul 2021 15:43:01 +0800 Subject: [PATCH 140/156] =?UTF-8?q?=E8=B0=83=E6=95=B42.1=E5=88=86=E6=94=AF?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E5=AE=A1=E6=A0=B8=E4=BA=BA=E5=91=98=20(#3389?= =?UTF-8?q?0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将关于approve相关的修改cherry-pick到2.1分支 --- tools/check_api_approvals.sh | 8 ++++---- tools/check_file_diff_approvals.sh | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/check_api_approvals.sh b/tools/check_api_approvals.sh index eb05468eda6ca..97d97e8c0a26a 100644 --- a/tools/check_api_approvals.sh +++ b/tools/check_api_approvals.sh @@ -39,16 +39,16 @@ function add_failed(){ api_spec_diff=`python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec.api ${PADDLE_ROOT}/paddle/fluid/API_PR.spec.api` if [ "$api_spec_diff" != "" ]; then - echo_line="You must have one RD (XiaoguangHu01 or lanxianghit) and one TPM (saxon-zh or jzhang533 or swtkiwi or Heeenrrry or TCChenlong) approval for the api change for the management reason of API interface.\n" + echo_line="You must have one RD (XiaoguangHu01 or lanxianghit) and one TPM (saxon-zh or jzhang533 or dingjiaweiww or Heeenrrry or TCChenlong) approval for the api change for the management reason of API interface.\n" check_approval 1 46782768 47554610 echo_line="" - check_approval 1 2870059 29231 27208573 28379894 11935832 + check_approval 1 2870059 29231 23093488 28379894 11935832 fi api_doc_spec_diff=`python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec.doc ${PADDLE_ROOT}/paddle/fluid/API_PR.spec.doc` if [ "$api_doc_spec_diff" != "" ]; then - echo_line="You must have one TPM (saxon-zh or jzhang533 or swtkiwi or Heeenrrry or TCChenlong) approval for the api change for the management reason of API document.\n" - check_approval 1 2870059 29231 27208573 28379894 11935832 + echo_line="You must have one TPM (saxon-zh or jzhang533 or dingjiaweiww or Heeenrrry or TCChenlong) approval for the api change for the management reason of API document.\n" + check_approval 1 2870059 29231 23093488 28379894 11935832 fi api_spec_diff=`python ${PADDLE_ROOT}/tools/check_api_source_without_core_ops.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.source.md5 ${PADDLE_ROOT}/paddle/fluid/API_PR.source.md5` diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh index ef9af288fb0a2..92e59675dad16 100644 --- a/tools/check_file_diff_approvals.sh +++ b/tools/check_file_diff_approvals.sh @@ -97,7 +97,7 @@ for API_FILE in ${API_FILES[*]}; do if [ "${API_CHANGE}" ] && [ "${GIT_PR_ID}" != "" ]; then # NOTE: per_page=10000 should be ok for all cases, a PR review > 10000 is not human readable. # You can use http://caius.github.io/github_id/ to find Github user id. - # approval_user_list: XiaoguangHu01 46782768,Xreki 12538138,luotao1 6836917,qingqing01 7845005,guoshengCS 14105589,heavengate 12605721,kuke 3064195,Superjomn 328693,lanxianghit 47554610,cyj1986 39645414,hutuxian 11195205,frankwhzhang 20274488,nepeplwu 45024560,Dianhai 38231817,chenwhql 22561442,zhiqiu 6888866,seiriosPlus 5442383,gongweibao 10721757,saxon-zh 2870059, zhouwei25 52485244, Aurelius84 9301846, liym27 33742067, zhhsplendid 7913861, kolinwei 22165420, liuwei1031 46661762, swtkiwi 27208573, juncaipeng 52520497, zhangting2020 26615455, Shixiaowei02 39303645, Heeenrrry 28379894,XieYunshen 32428676, Dong Daxiang 35550832, phlrain 43953930. + # approval_user_list: XiaoguangHu01 46782768,Xreki 12538138,luotao1 6836917,qingqing01 7845005,guoshengCS 14105589,heavengate 12605721,kuke 3064195,Superjomn 328693,lanxianghit 47554610,cyj1986 39645414,hutuxian 11195205,frankwhzhang 20274488,nepeplwu 45024560,Dianhai 38231817,chenwhql 22561442,zhiqiu 6888866,seiriosPlus 5442383,gongweibao 10721757,saxon-zh 2870059, zhouwei25 52485244, Aurelius84 9301846, liym27 33742067, zhhsplendid 7913861, kolinwei 22165420, liuwei1031 46661762, dingjiaweiww 23093488, juncaipeng 52520497, zhangting2020 26615455, Shixiaowei02 39303645, Heeenrrry 28379894,XieYunshen 32428676, Dong Daxiang 35550832, phlrain 43953930. if [ "${API_FILE}" == "CMakeLists.txt" ];then echo_line="You must have one RD (wanghuancoder, luotao1 or XiaoguangHu01) approval for CMakeLists.txt, which manages the compilation parameter.\n" check_approval 1 6836917 46782768 26922892 @@ -105,8 +105,8 @@ for API_FILE in ${API_FILES[*]}; do echo_line="You must have one RD (lanxianghit (Recommend), phlrain or luotao1) approval for the python/paddle/fluid/init.py, which manages the environment variables.\n" check_approval 1 6836917 47554610 43953930 elif [ "${API_FILE}" == "python/requirements.txt" ];then - echo_line="You must have one RD (phlrain) and one TPM (swtkiwi) and one QA (kolinwei) approval for python/requirements.txt, which manages the third-party python package.\n" - check_approval 3 43953930 27208573 22165420 + echo_line="You must have one RD (phlrain) and one TPM (dingjiaweiww) and one QA (kolinwei) approval for python/requirements.txt, which manages the third-party python package.\n" + check_approval 3 43953930 23093488 22165420 elif [ "${API_FILE}" == "paddle/fluid/operators/distributed/send_recv.proto.in" ];then echo_line="You must have one RD (gongweibao or seiriosPlus) approval for the paddle/fluid/operators/distributed/send_recv.proto.in, which manages the environment variables.\n" check_approval 1 10721757 5442383 From fe827540a78de6bd8601a52526ec74ba26ae8fa8 Mon Sep 17 00:00:00 2001 From: Wilber Date: Mon, 5 Jul 2021 16:43:40 +0800 Subject: [PATCH 141/156] cherry-pick prs. (#33932) --- cmake/inference_lib.cmake | 2 +- cmake/tensorrt.cmake | 20 ++++++++++++++++++- .../inference/api/paddle_analysis_config.h | 2 +- tools/remove_grad_op_and_kernel.py | 7 ++++--- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 9574af761ed10..2e7d32046fdec 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -345,7 +345,7 @@ function(version version_file) file(APPEND ${version_file} "CXX compiler version: ${CMAKE_CXX_COMPILER_VERSION}\n") if(TENSORRT_FOUND) file(APPEND ${version_file} - "WITH_TENSORRT: ${TENSORRT_FOUND}\n" "TensorRT version: v${TENSORRT_MAJOR_VERSION}\n") + "WITH_TENSORRT: ${TENSORRT_FOUND}\n" "TensorRT version: v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION}\n") endif() if(WITH_LITE) file(APPEND ${version_file} "WITH_LITE: ${WITH_LITE}\n" "LITE_GIT_TAG: ${LITE_GIT_TAG}\n") diff --git a/cmake/tensorrt.cmake b/cmake/tensorrt.cmake index 889332fc55704..e4b22befff850 100644 --- a/cmake/tensorrt.cmake +++ b/cmake/tensorrt.cmake @@ -47,11 +47,23 @@ if(TENSORRT_FOUND) file(READ ${TENSORRT_INCLUDE_DIR}/NvInfer.h TENSORRT_VERSION_FILE_CONTENTS) string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)" TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}") + string(REGEX MATCH "define NV_TENSORRT_MINOR +([0-9]+)" TENSORRT_MINOR_VERSION + "${TENSORRT_VERSION_FILE_CONTENTS}") + string(REGEX MATCH "define NV_TENSORRT_PATCH +([0-9]+)" TENSORRT_PATCH_VERSION + "${TENSORRT_VERSION_FILE_CONTENTS}") + string(REGEX MATCH "define NV_TENSORRT_BUILD +([0-9]+)" TENSORRT_BUILD_VERSION + "${TENSORRT_VERSION_FILE_CONTENTS}") if("${TENSORRT_MAJOR_VERSION}" STREQUAL "") file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h TENSORRT_VERSION_FILE_CONTENTS) string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)" TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}") + string(REGEX MATCH "define NV_TENSORRT_MINOR +([0-9]+)" TENSORRT_MINOR_VERSION + "${TENSORRT_VERSION_FILE_CONTENTS}") + string(REGEX MATCH "define NV_TENSORRT_PATCH +([0-9]+)" TENSORRT_PATCH_VERSION + "${TENSORRT_VERSION_FILE_CONTENTS}") + string(REGEX MATCH "define NV_TENSORRT_BUILD +([0-9]+)" TENSORRT_BUILD_VERSION + "${TENSORRT_VERSION_FILE_CONTENTS}") endif() if("${TENSORRT_MAJOR_VERSION}" STREQUAL "") @@ -60,9 +72,15 @@ if(TENSORRT_FOUND) string(REGEX REPLACE "define NV_TENSORRT_MAJOR +([0-9]+)" "\\1" TENSORRT_MAJOR_VERSION "${TENSORRT_MAJOR_VERSION}") + string(REGEX REPLACE "define NV_TENSORRT_MINOR +([0-9]+)" "\\1" + TENSORRT_MINOR_VERSION "${TENSORRT_MINOR_VERSION}") + string(REGEX REPLACE "define NV_TENSORRT_PATCH +([0-9]+)" "\\1" + TENSORRT_PATCH_VERSION "${TENSORRT_PATCH_VERSION}") + string(REGEX REPLACE "define NV_TENSORRT_BUILD +([0-9]+)" "\\1" + TENSORRT_BUILD_VERSION "${TENSORRT_BUILD_VERSION}") message(STATUS "Current TensorRT header is ${TENSORRT_INCLUDE_DIR}/NvInfer.h. " - "Current TensorRT version is v${TENSORRT_MAJOR_VERSION}. ") + "Current TensorRT version is v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION} ") include_directories(${TENSORRT_INCLUDE_DIR}) link_directories(${TENSORRT_LIBRARY}) add_definitions(-DPADDLE_WITH_TENSORRT) diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index f9e4869934a0f..6e986f5f4822b 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -678,7 +678,7 @@ struct PD_INFER_DECL AnalysisConfig { bool xpu_adaptive_seqlen_; // mkldnn related. - int mkldnn_cache_capacity_{0}; + int mkldnn_cache_capacity_{10}; bool use_mkldnn_quantizer_{false}; std::shared_ptr mkldnn_quantizer_config_; bool use_mkldnn_bfloat16_{false}; diff --git a/tools/remove_grad_op_and_kernel.py b/tools/remove_grad_op_and_kernel.py index 85bbf8cdddc29..e8ab321e96105 100644 --- a/tools/remove_grad_op_and_kernel.py +++ b/tools/remove_grad_op_and_kernel.py @@ -20,6 +20,7 @@ import sys import re import glob +import io def find_type_files(cur_dir, file_type, file_list=[]): @@ -124,7 +125,7 @@ def update_operator_cmake(cmake_file): custom_pattern2 = custom_pattern2[:-1] all_matches = [] - with open(op_file, 'r') as f: + with io.open(op_file, 'r', encoding='utf-8') as f: content = ''.join(f.readlines()) op, op_count = remove_grad_op_and_kernel(content, op_pattern1, @@ -157,8 +158,8 @@ def update_operator_cmake(cmake_file): for i in all_matches: content = content.replace(i, '') - with open(op_file, 'w') as f: - f.write(content) + with io.open(op_file, 'w', encoding='utf-8') as f: + f.write(u'{}'.format(content)) # 2. update operators/CMakeLists.txt cmake_file = os.path.join(tool_dir, From 0d6c7532be2e5e44edc1326aa7a22bcb261b31ac Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Tue, 6 Jul 2021 15:57:55 +0800 Subject: [PATCH 142/156] [Cherry-pick][Dy2Stat] Fix unique_name in create_static_variable_gast_node (#33963) (#33980) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug fixes， [Cherry-pick][Dy2Stat] Fix unique_name in create_static_variable_gast_node (#33963) --- .../fluid/dygraph/dygraph_to_static/variable_trans_func.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py index 673d30cffbe1e..eb79139406908 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py @@ -18,6 +18,7 @@ import gast from paddle.fluid import core +from paddle.fluid import unique_name from paddle.fluid.framework import Variable from paddle.fluid.layers import fill_constant from paddle.fluid.layer_helper import LayerHelper @@ -84,7 +85,7 @@ def to_static_variable_gast_node(name): def create_static_variable_gast_node(name): func_code = "{} = paddle.jit.dy2static\ .data_layer_not_check(name='{}', shape=[-1], dtype='float32')".format( - name, name) + name, unique_name.generate(name)) return gast.parse(func_code).body[0] From 12f103aaa2cac504484a475efea9e2cb973203c6 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Wed, 7 Jul 2021 09:59:45 +0800 Subject: [PATCH 143/156] [Cherry-Pick 33556]del python2 code (#33987) * del python2 code * cherry-pick 33556 --- CMakeLists.txt | 2 +- python/requirements.txt | 3 +-- python/unittest_py/requirements.txt | 6 ++---- tools/count_api_without_core_ops.py | 5 +---- tools/print_signatures.py | 7 ++----- tools/sampcd_processor.py | 2 +- 6 files changed, 8 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index edb9a46c03ab8..a4357f5d155d0 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -209,7 +209,7 @@ option(WITH_STRIP "Strip so files of Whl packages" OFF) # PY_VERSION if(NOT PY_VERSION) - set(PY_VERSION 2.7) + set(PY_VERSION 3.6) endif() set(PYBIND11_PYTHON_VERSION ${PY_VERSION}) diff --git a/python/requirements.txt b/python/requirements.txt index 14bd5e7caa6f5..e9da2aa24d6cb 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,4 @@ requests>=2.20.0 -numpy>=1.13, <=1.16.4 ; python_version<"3.5" numpy>=1.13 ; python_version>="3.5" and platform_system != "Windows" numpy>=1.13, <=1.19.3 ; python_version>="3.5" and platform_system == "Windows" protobuf>=3.1.0 @@ -7,5 +6,5 @@ gast>=0.3.3, <=0.4.0 ; platform_system != "Windows" gast==0.3.3 ; platform_system == "Windows" Pillow six -decorator==4.4.2 +decorator astor diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt index 752f3545c69cc..8fd1be69a3d7f 100644 --- a/python/unittest_py/requirements.txt +++ b/python/unittest_py/requirements.txt @@ -4,10 +4,8 @@ pycrypto ; platform_system != "Windows" mock gym opencv-python<=4.2.0.32 -visualdl ; python_version>="3.5" +visualdl paddle2onnx>=0.4 -scipy>=0.19.0, <=1.2.1 ; python_version<"3.5" -scipy<=1.3.1 ; python_version=="3.5" -scipy ; python_version>"3.5" +scipy prettytable distro diff --git a/tools/count_api_without_core_ops.py b/tools/count_api_without_core_ops.py index 664b94a059f5c..7af597600e001 100644 --- a/tools/count_api_without_core_ops.py +++ b/tools/count_api_without_core_ops.py @@ -37,10 +37,7 @@ def md5(doc): try: hashinst = hashlib.md5() - if platform.python_version()[0] == "2": - hashinst.update(str(doc)) - else: - hashinst.update(str(doc).encode('utf-8')) + hashinst.update(str(doc).encode('utf-8')) md5sum = hashinst.hexdigest() except UnicodeDecodeError as e: md5sum = None diff --git a/tools/print_signatures.py b/tools/print_signatures.py index 6de9d84379fea..be32ef09b70d6 100644 --- a/tools/print_signatures.py +++ b/tools/print_signatures.py @@ -36,10 +36,7 @@ def md5(doc): try: hashinst = hashlib.md5() - if platform.python_version()[0] == "2": - hashinst.update(str(doc)) - else: - hashinst.update(str(doc).encode('utf-8')) + hashinst.update(str(doc).encode('utf-8')) md5sum = hashinst.hexdigest() except UnicodeDecodeError as e: md5sum = None @@ -142,7 +139,7 @@ def visit_member(parent_name, member, member_name=None): def is_primitive(instance): - int_types = (int, long) if platform.python_version()[0] == "2" else (int, ) + int_types = (int, ) pritimitive_types = int_types + (float, str) if isinstance(instance, pritimitive_types): return True diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index a1658e3c2edf7..f243ada073634 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -389,7 +389,7 @@ def execute_samplecode(tfname): """ result = True msg = None - if platform.python_version()[0] in ["2", "3"]: + if platform.python_version()[0] in ["3"]: cmd = [sys.executable, tfname] else: logger.error("Error: fail to parse python version!") From f2f2fd80735fef0c4eb639cc22e1279164377259 Mon Sep 17 00:00:00 2001 From: Jacek Czaja Date: Fri, 9 Jul 2021 10:49:59 +0200 Subject: [PATCH 144/156] [oneDNN] Fix to #33282 , added support of X input broadcasting to oneDNN elementwise ops (#33549) (#33845) * - fix to #33282 * - Increased threshold for elementwise_mul_bf16 grad * -disabled faulty UT * - fix to approval --- .../framework/ir/graph_pattern_detector.cc | 11 +----- .../ir/mkldnn/mkldnn_inplace_pass_tester.cc | 2 +- .../mkldnn/elementwise_mkldnn_op.h | 14 +------ .../operators/mkldnn/test_mkldnn_caching.cc | 12 ------ .../mkldnn/test_mkldnn_op_inplace.cc | 6 --- paddle/fluid/platform/mkldnn_reuse.h | 27 ++++++------- .../mkldnn/test_elementwise_add_mkldnn_op.py | 20 ++++++++++ .../test_elementwise_mul_bf16_mkldnn_op.py | 38 ++++++++++--------- .../mkldnn/test_elementwise_mul_mkldnn_op.py | 10 +++++ 9 files changed, 67 insertions(+), 73 deletions(-) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index 064da3d941602..8caec54bdffb4 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -2340,16 +2340,7 @@ PDNode *patterns::DuplicatedInputs::operator()() { PDNode *patterns::MKLDNNInPlace::operator()() { const std::unordered_set &supported_op_types = { - "abs", - "elementwise_mul", - "elementwise_add", - "gelu", - "leaky_relu", - "relu", - "softmax", - "sqrt", - "swish", - "tanh"}; + "abs", "gelu", "leaky_relu", "relu", "softmax", "sqrt", "swish", "tanh"}; auto possible_inplace_op = pattern->NewNode(inplace_to_be_op_repr()) ->assert_is_ops(supported_op_types); diff --git a/paddle/fluid/framework/ir/mkldnn/mkldnn_inplace_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/mkldnn_inplace_pass_tester.cc index 01abe5a8d281b..90dc780113107 100644 --- a/paddle/fluid/framework/ir/mkldnn/mkldnn_inplace_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/mkldnn_inplace_pass_tester.cc @@ -167,7 +167,7 @@ TEST(MKLDNNInplacePass, inplace_softmax_branched) { TEST(MKLDNNInplacePass, inplace_elementwise_add) { // Two elementwise_add mkl-dnn enabled op instances to be made inplace - MKLDNNInplacePassTest().MainTest("elementwise_add", false, 1); + MKLDNNInplacePassTest().MainTest("elementwise_add", false, 0); } TEST(MKLDNNInplacePass, inplace_tanh) { MKLDNNInplacePassTest().MainTest("tanh", false, 1); diff --git a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h index e5d20893335f7..ddad70a6a5f31 100644 --- a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h +++ b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h @@ -47,23 +47,13 @@ class EltwiseMKLDNNKernel : public framework::OpKernel { float scale_o = ctx.Attr("Scale_out"); int axis = ctx.Attr("axis"); - bool is_inplaced = x->IsSharedBufferWith(*z); - - std::string key = is_inplaced - ? platform::CreateKey(dev_ctx, ctx.OutputName("Out"), - x->format(), y->format()) - : ctx.OutputName("Out"); - platform::BinaryMKLDNNHandler handler( BINARY_OP, axis, dev_ctx, mkldnn_engine, ctx.GetPlace(), x, y, z, - scale_x, scale_y, scale_o, key); + scale_x, scale_y, scale_o, ctx.OutputName("Out")); const auto src_x_memory = handler.AcquireSrcMemory(x); const auto src_y_memory = handler.AcquireSecondSrcMemory(y); - - // For Inplace src and and dst are the same memory object - const auto dst_memory = - is_inplaced ? src_x_memory : handler.AcquireDstMemory(z); + const auto dst_memory = handler.AcquireDstMemory(z); const auto binary_prim = handler.AcquireForwardPrimitive(); diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc b/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc index d6cd76b697f51..cad4f47ec1402 100644 --- a/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc +++ b/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc @@ -180,17 +180,5 @@ TEST(test_elementwise_add_reuse_cache, cpu_place) { "Wrong number of cached oneDNN objects")); } -TEST(test_elementwises_sequence_reuse_cache, cpu_place) { - framework::DDim dims({32, 64}); - platform::CPUPlace p; - CacheTester ct; - RunOperator(p, "elementwise_add", dims, "elementwise_add_out", true); - RunOperator(p, "elementwise_mul", dims, "elementwise_add_out", true); - RunOperator(p, "relu", dims, "elementwise_add_out", true); - PADDLE_ENFORCE_EQ(ct.Analyze(11), true, - platform::errors::InvalidArgument( - "Wrong number of cached oneDNN objects")); -} - } // namespace operators } // namespace paddle diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc b/paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc index 643de3fd5be70..0612417c46ce3 100644 --- a/paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc +++ b/paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc @@ -128,12 +128,6 @@ TEST(test_softmax_inplace, cpu_place) { ASSERT_TRUE(TestMain(p, "softmax", dims, 1)); } -TEST(test_elementwise_add_inplace, cpu_place) { - framework::DDim dims({1, 12, 20, 20}); - platform::CPUPlace p; - ASSERT_TRUE(TestMain(p, "elementwise_add", dims, 2)); -} - TEST(test_relu_inplace, cpu_place) { framework::DDim dims({1, 12, 20, 20}); platform::CPUPlace p; diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index f1eb1f9636375..95d04e9822f17 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -538,17 +538,8 @@ class BinaryMKLDNNHandler : public platform::MKLDNNHandlerT { const std::string& uniq_name) : platform::MKLDNNHandlerT( dev_ctx, engine, cpu_place, - platform::CreateKey( - dev_ctx, framework::vectorize(x->dims()), uniq_name, - (algo == dnnl::algorithm::binary_mul ? "M" : ""))) { - // bradcasting combined with in-place may require - auto rankdiff = x->dims().size() - y->dims().size(); - if (rankdiff > 0) { - auto suffix = std::to_string(rankdiff); - this->key_ += suffix; - this->key_common_ += suffix; - } - + platform::CreateKey(dev_ctx, framework::vectorize(x->dims()), + uniq_name)) { if (!this->isCached()) { PADDLE_ENFORCE_EQ( x->layout(), DataLayout::kMKLDNN, @@ -568,18 +559,24 @@ class BinaryMKLDNNHandler : public platform::MKLDNNHandlerT { const auto src_y_tz = framework::vectorize(y->dims()); // if output tensor(z) is nullptr then we are computing into oneDNN // managed buffer - const auto dst_tz = - (z == nullptr) ? src_x_tz : framework::vectorize(z->dims()); + auto rankdiff = x->dims().size() - y->dims().size(); + const auto dst_tz = (z == nullptr) ? (rankdiff > 0 ? src_x_tz : src_y_tz) + : framework::vectorize(z->dims()); - const auto src0_md = dnnl::memory::desc( + auto src0_md = dnnl::memory::desc( src_x_tz, platform::MKLDNNGetDataType(), x->format()); auto src1_md = dnnl::memory::desc( src_y_tz, platform::MKLDNNGetDataType(), y->format()); - if (rankdiff > 0) { + if (rankdiff > 0) { // Second input is of smaller rank than first std::vector dims1_ex(rankdiff, 1); dims1_ex.insert(next(dims1_ex.begin(), (axis == -1 ? rankdiff : axis)), src_y_tz.begin(), src_y_tz.end()); src1_md = src1_md.reshape(dims1_ex); + } else if (rankdiff < 0) { // First input is of smaller than second + std::vector dims0_ex(-rankdiff, 1); + dims0_ex.insert(next(dims0_ex.begin(), (axis == -1 ? -rankdiff : axis)), + src_x_tz.begin(), src_x_tz.end()); + src0_md = src0_md.reshape(dims0_ex); } const auto dst_md = memory::desc(dst_tz, platform::MKLDNNGetDataType(), MKLDNNMemoryFormat::any); diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py index 28456a3e91dca..585ae38875cc7 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py @@ -73,6 +73,26 @@ def init_axis(self): self.axis = 1 +class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestMKLDNNElementwiseAddOp): + def init_input_output(self): + self.x = np.random.rand(10, 12).astype(self.dtype) + self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) + self.out = self.x + self.y + + def init_axis(self): + self.axis = 2 + + # TODO(jczaja): Enable when grad is ready + def test_check_grad_normal(self): + pass + + def test_check_grad_ingore_y(self): + pass + + def test_check_grad_ingore_x(self): + pass + + ''' INT8 Tests ''' diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py index 9b7f4b9b860de..b67ae17ba3a5a 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py @@ -85,26 +85,30 @@ def compute_reduced_gradients(self, out_grads): part_sum = np.add.reduceat(part_sum, [0], axis=2) return part_sum.flatten() + # TODO(jczaja): elementwise_mul bf16 grad got some potential + # accuracy problems that need to be explained def test_check_grad_normal(self): - self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], - "Out", - check_dygraph=False, - user_defined_grads=[ - np.multiply(self.x, self.y), - self.compute_reduced_gradients(np.multiply(self.x, self.x)) - ], - user_defined_grad_outputs=[self.x_bf16]) + pass + #self.check_grad_with_place( + # core.CPUPlace(), ["X", "Y"], + # "Out", + # check_dy_graph=False, + # user_defined_grads=[ + # np.multiply(self.x, self.y), + # self.compute_reduced_gradients(np.multiply(self.x, self.x)) + # ], + # user_defined_grad_outputs=[self.x_bf16]) def test_check_grad_ingore_x(self): - self.check_grad_with_place( - core.CPUPlace(), ["Y"], - "Out", - check_dygraph=False, - user_defined_grads=[ - self.compute_reduced_gradients(np.multiply(self.x, self.x)) - ], - user_defined_grad_outputs=[self.x_bf16]) + pass + #self.check_grad_with_place( + # core.CPUPlace(), ["Y"], + # "Out", + # check_dy_graph=False, + # user_defined_grads=[ + # self.compute_reduced_gradients(np.multiply(self.x, self.x)) + # ], + # user_defined_grad_outputs=[self.x_bf16]) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py index 03dc2421b65b0..f2648e5b723ed 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py @@ -62,6 +62,16 @@ def init_input_output(self): self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) self.out = np.multiply(self.x, self.y) + # TODO(jczaja): Enable when grad is ready + def test_check_grad_normal(self): + pass + + def test_check_grad_ingore_y(self): + pass + + def test_check_grad_ingore_x(self): + pass + ''' INT8 Tests ''' From 8417ad60c0af7f6ba155445baf188d584a46630e Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Fri, 9 Jul 2021 17:59:01 +0800 Subject: [PATCH 145/156] [Cherry-pick] Up cxx11 check to cxx14 (#34015) (#34034) [Cherry-pick] Up cxx11 check to cxx14 #34034 --- paddle/fluid/extension/include/ext_all.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/extension/include/ext_all.h b/paddle/fluid/extension/include/ext_all.h index f2b3bcf5191c3..6987b33012f64 100644 --- a/paddle/fluid/extension/include/ext_all.h +++ b/paddle/fluid/extension/include/ext_all.h @@ -14,8 +14,8 @@ limitations under the License. */ #pragma once -#if !defined(_MSC_VER) && __cplusplus < 199711L -#error C++11 or later compatible compiler is required to use Paddle. +#if !defined(_MSC_VER) && __cplusplus < 201402L +#error C++14 or later compatible compiler is required to use Paddle. #endif #ifdef _WIN32 From ed7903cda7e2e8ce44f4da02f7e1cf70f1de37a1 Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Fri, 9 Jul 2021 19:08:00 +0800 Subject: [PATCH 146/156] make DataLoader warning less noisy. test=develop (#34001) --- python/paddle/fluid/dataloader/fetcher.py | 24 +++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/python/paddle/fluid/dataloader/fetcher.py b/python/paddle/fluid/dataloader/fetcher.py index 05382b04dc457..8ccec81810a0a 100644 --- a/python/paddle/fluid/dataloader/fetcher.py +++ b/python/paddle/fluid/dataloader/fetcher.py @@ -14,8 +14,9 @@ import logging from ..log_helper import get_logger +from collections.abc import Sequence, Mapping -from collections.abc import Sequence +_WARNING_TO_LOG = True class _DatasetFetcher(object): @@ -24,13 +25,17 @@ def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): self.auto_collate_batch = auto_collate_batch self.collate_fn = collate_fn self.drop_last = drop_last - self._is_warning_logged = False def fetch(self, batch_indices): raise NotImplementedError("'fetch' not implement for class {}".format( self.__class__.__name__)) def _log_warning(self): + # only log warning on GPU 0 when distributed launch + from ...distributed import get_world_size, get_rank + if get_world_size() >= 2 and get_rank() != 0: + return + warn_str = "Detect dataset only contains single fileds, return format " \ "changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add " \ "a list surround output data(e.g. return [data]), and in " \ @@ -77,10 +82,12 @@ def fetch(self, batch_indices): if len(data) == 0 or (self.drop_last and len(data) < len(batch_indices)): raise StopIteration - if not isinstance(data[0], - Sequence) and not self._is_warning_logged: + + global _WARNING_TO_LOG + if not isinstance(data[0], (Sequence, Mapping)) \ + and _WARNING_TO_LOG: self._log_warning() - self._is_warning_logged = True + _WARNING_TO_LOG = False else: data = next(self.dataset_iter) @@ -98,10 +105,11 @@ def fetch(self, batch_indices): if self.auto_collate_batch: data = [self.dataset[idx] for idx in batch_indices] - if not isinstance(data[0], - Sequence) and not self._is_warning_logged: + global _WARNING_TO_LOG + if not isinstance(data[0], (Sequence, Mapping)) \ + and _WARNING_TO_LOG: self._log_warning() - self._is_warning_logged = True + _WARNING_TO_LOG = False else: data = self.dataset[batch_indices] From 0f266ac18bcac01bd0438e4c4b95ff79237eda6b Mon Sep 17 00:00:00 2001 From: taixiurong Date: Mon, 12 Jul 2021 10:38:50 +0800 Subject: [PATCH 147/156] cherry pick xpu to 2.1 (#34000) * update xpu cmake for kunlun (#33328) * xpu support amp (#33809) * fix bug DLTP-31078 (#33877) * update xpu cmake (#33906) * [xpu] add dropout & amp ops in xpu place (#33891) Co-authored-by: TTerror --- cmake/external/xpu.cmake | 15 +- paddle/fluid/imperative/amp_auto_cast.cc | 6 +- .../amp/check_finite_and_unscale_op_xpu.cc | 170 ++++++++++++ .../amp/update_loss_scaling_op_xpu.cc | 166 ++++++++++++ paddle/fluid/operators/cast_op_xpu.cc | 15 +- paddle/fluid/operators/dropout_op_xpu.cc | 175 ++++++------- .../elementwise/elementwise_add_op_xpu.cc | 20 ++ paddle/fluid/operators/matmul_op_xpu.cc | 81 +++--- paddle/fluid/operators/matmul_v2_op_xpu.cc | 79 +++--- paddle/fluid/operators/softmax_op_xpu.cc | 4 +- .../softmax_with_cross_entropy_op_xpu.cc | 5 +- paddle/fluid/platform/xpu_header.h | 15 +- paddle/fluid/pybind/pybind.cc | 4 +- .../contrib/mixed_precision/fp16_lists.py | 10 +- python/paddle/fluid/dygraph/amp/auto_cast.py | 5 +- .../paddle/fluid/dygraph/amp/loss_scaler.py | 5 +- .../test_amp_check_finite_and_scale_op_xpu.py | 99 +++++++ .../unittests/xpu/test_dropout_op_xpu.py | 6 +- .../xpu/test_update_loss_scaling_op_xpu.py | 245 ++++++++++++++++++ 19 files changed, 938 insertions(+), 187 deletions(-) create mode 100644 paddle/fluid/operators/amp/check_finite_and_unscale_op_xpu.cc create mode 100644 paddle/fluid/operators/amp/update_loss_scaling_op_xpu.cc create mode 100644 python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py create mode 100644 python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index a8c33618a6135..8d202b5a99bfc 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -27,19 +27,18 @@ ELSEIF(WITH_CENTOS) SET(XPU_XRE_DIR_NAME "xre-centos7_x86_64") SET(XPU_XDNN_DIR_NAME "xdnn-centos7_x86_64") SET(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64") + ELSE () SET(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64") SET(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64") SET(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64") ENDIF() -IF(NOT XPU_BASE_URL) - SET(XPU_BASE_URL "https://baidu-kunlun-product.cdn.bcebos.com/KL-SDK/klsdk-dev/20210527") -ENDIF() - +SET(XPU_BASE_URL_WITHOUT_DATE "https://baidu-kunlun-product.cdn.bcebos.com/KL-SDK/klsdk-dev") +SET(XPU_BASE_URL "${XPU_BASE_URL_WITHOUT_DATE}/20210701") SET(XPU_XRE_URL "${XPU_BASE_URL}/${XPU_XRE_DIR_NAME}.tar.gz" CACHE STRING "" FORCE) SET(XPU_XDNN_URL "${XPU_BASE_URL}/${XPU_XDNN_DIR_NAME}.tar.gz" CACHE STRING "" FORCE) -SET(XPU_XCCL_URL "${XPU_BASE_URL}/${XPU_XCCL_DIR_NAME}.tar.gz" CACHE STRING "" FORCE) +SET(XPU_XCCL_URL "${XPU_BASE_URL_WITHOUT_DATE}/20210623/${XPU_XCCL_DIR_NAME}.tar.gz" CACHE STRING "" FORCE) SET(XPU_PACK_DEPENCE_URL "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/pack_paddle_depence.sh" CACHE STRING "" FORCE) SET(XPU_SOURCE_DIR "${THIRD_PARTY_PATH}/xpu") @@ -96,7 +95,11 @@ ELSE(WITH_XPU_BKCL) TARGET_LINK_LIBRARIES(xpulib ${XPU_API_LIB} ${XPU_RT_LIB}) ENDIF(WITH_XPU_BKCL) -ADD_DEPENDENCIES(xpulib ${XPU_PROJECT}) +if(NOT XPU_SDK_ROOT) + ADD_DEPENDENCIES(xpulib ${XPU_PROJECT}) +else() + ADD_CUSTOM_TARGET(extern_xpu DEPENDS xpulib) +endif() # Ensure that xpu/api.h can be included without dependency errors. file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/.xpu_headers_dummy.cc CONTENT "") diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc index b4154737e0fbc..d67a548315541 100644 --- a/paddle/fluid/imperative/amp_auto_cast.cc +++ b/paddle/fluid/imperative/amp_auto_cast.cc @@ -33,7 +33,8 @@ AmpOperators::AmpOperators() for (auto it = all_kernels.begin(); it != all_kernels.end(); it++) { bool supported = false; for (auto& kernel_type : it->second) { - if (platform::is_gpu_place(kernel_type.first.place_) && + if ((platform::is_gpu_place(kernel_type.first.place_) || + platform::is_xpu_place(kernel_type.first.place_)) && kernel_type.first.data_type_ == fp16_dtype) { supported = true; } @@ -91,7 +92,8 @@ inline std::string GetDtypeStr( inline bool NeedCast(const std::shared_ptr& var) { if (platform::is_gpu_place(var->Place()) || - platform::is_cuda_pinned_place(var->Place())) { + platform::is_cuda_pinned_place(var->Place()) || + platform::is_xpu_place(var->Place())) { // CudaPinndePlace is added for varbase created by dataloader if (var->DataType() == framework::proto::VarType::FP32 || var->DataType() == framework::proto::VarType::FP16) { diff --git a/paddle/fluid/operators/amp/check_finite_and_unscale_op_xpu.cc b/paddle/fluid/operators/amp/check_finite_and_unscale_op_xpu.cc new file mode 100644 index 0000000000000..210f3e098f95f --- /dev/null +++ b/paddle/fluid/operators/amp/check_finite_and_unscale_op_xpu.cc @@ -0,0 +1,170 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef PADDLE_WITH_XPU +#include "paddle/fluid/operators/amp/check_finite_and_unscale_op.h" +#include "paddle/fluid/operators/amp/fp16_type_traits.h" +#include "paddle/fluid/platform/float16.h" +namespace paddle { +namespace operators { +template +class CheckFiniteAndUnscaleXPUKernel : public framework::OpKernel { + using MPDType = typename details::MPTypeTrait::Type; + using XPUTyp = typename XPUTypeTrait::Type; + + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto& dev_ctx = ctx.template device_context(); + const auto xs = ctx.MultiInput("X"); + const auto* scale = ctx.Input("Scale"); + auto outs = ctx.MultiOutput("Out"); + auto* found_inf = ctx.Output("FoundInfinite"); + + const MPDType* scale_data = scale->data(); + bool* found_inf_data = found_inf->mutable_data(dev_ctx.GetPlace()); + + // cpy to cpu + bool cpu_found_inf_data = false; + + MPDType cpu_scale_data; + if (platform::is_xpu_place(scale->place())) { + xpu_memcpy(&cpu_scale_data, scale_data, sizeof(MPDType), + XPUMemcpyKind::XPU_DEVICE_TO_HOST); + } else { + cpu_scale_data = (*scale_data); + } + MPDType inverse_scale = 1.0 / cpu_scale_data; + for (size_t i = 0; i < xs.size(); ++i) { + const auto* x = xs[i]; + auto* out = outs[i]; + out->mutable_data(dev_ctx.GetPlace()); + framework::Tensor is_finite = + ctx.AllocateTmpTensor(x->dims(), + dev_ctx); + framework::Tensor is_nan = + ctx.AllocateTmpTensor(x->dims(), + dev_ctx); + framework::Tensor is_finite_and_nan = + ctx.AllocateTmpTensor(x->dims(), + dev_ctx); + if (cpu_found_inf_data == false) { + int r = xpu::isfinite(dev_ctx.x_context(), + reinterpret_cast(x->data()), + is_finite.data(), x->numel()); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(isfinite) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + r = xpu::logical_not(dev_ctx.x_context(), reinterpret_cast( + is_finite.data()), + is_finite.data(), x->numel()); + PADDLE_ENFORCE_EQ( + r, XPU_SUCCESS, + platform::errors::External("XPU API(logical_not) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + r = xpu::isnan(dev_ctx.x_context(), + reinterpret_cast(x->data()), + is_nan.data(), x->numel()); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(isnan) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + r = xpu::logical_or(dev_ctx.x_context(), is_finite.data(), + is_nan.data(), is_finite.data(), + x->numel()); + PADDLE_ENFORCE_EQ( + r, XPU_SUCCESS, + platform::errors::External("XPU API(logical_or) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + r = xpu::any(dev_ctx.x_context(), is_finite.data(), + found_inf_data, x->numel()); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(any) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + memory::Copy(platform::CPUPlace(), &cpu_found_inf_data, + BOOST_GET_CONST(platform::XPUPlace, dev_ctx.GetPlace()), + found_inf_data, sizeof(bool)); + } + + if (cpu_found_inf_data) { + inverse_scale = 0.0; + } + auto dev_env = XPUEnv::getenv("XPUSIM_DEVICE_MODEL"); + + if (std::is_same::value && + (dev_env == nullptr || std::strcmp(dev_env, "KUNLUN1"))) { + framework::Tensor float_x; + framework::Tensor float_out; + float_x.mutable_data(dev_ctx.GetPlace(), + x->numel() * sizeof(MPDType)); + float_out.mutable_data(dev_ctx.GetPlace(), + out->numel() * sizeof(MPDType)); + int r = xpu::cast_v2(dev_ctx.x_context(), + reinterpret_cast(x->data()), + float_x.data(), x->numel()); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(cast_v2) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + + r = xpu::scale(dev_ctx.x_context(), float_x.data(), + float_out.data(), x->numel(), false, + inverse_scale, 0.0); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(scale) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + + r = xpu::cast_v2(dev_ctx.x_context(), float_out.data(), + reinterpret_cast(out->data()), + out->numel()); + + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(cast_v2) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + if (dev_ctx.x_context()->xpu_stream) { + dev_ctx.Wait(); + } + + } else { + int r = xpu::scale(dev_ctx.x_context(), + reinterpret_cast(x->data()), + reinterpret_cast(out->data()), + x->numel(), false, inverse_scale, 0.0); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(scale) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + } + } + memory::Copy(BOOST_GET_CONST(platform::XPUPlace, dev_ctx.GetPlace()), + found_inf_data, platform::CPUPlace(), &cpu_found_inf_data, + sizeof(bool)); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +namespace plat = paddle::platform; +REGISTER_OP_XPU_KERNEL(check_finite_and_unscale, + ops::CheckFiniteAndUnscaleXPUKernel, + ops::CheckFiniteAndUnscaleXPUKernel); + +#endif diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op_xpu.cc b/paddle/fluid/operators/amp/update_loss_scaling_op_xpu.cc new file mode 100644 index 0000000000000..1f05e5f246d9c --- /dev/null +++ b/paddle/fluid/operators/amp/update_loss_scaling_op_xpu.cc @@ -0,0 +1,166 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef PADDLE_WITH_XPU +#include "paddle/fluid/operators/amp/update_loss_scaling_op.h" +#include +#include +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/amp/fp16_type_traits.h" +#include "paddle/fluid/platform/float16.h" + +namespace paddle { +namespace operators { + +template +class UpdateLossScalingXPUKernel : public framework::OpKernel { + using MPDType = typename details::MPTypeTrait::Type; + using XPUTyp = typename XPUTypeTrait::Type; + + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto& dev_ctx = ctx.template device_context(); + + const auto xs = ctx.MultiInput("X"); + auto outs = ctx.MultiOutput("Out"); + const auto* found_inf = ctx.Input("FoundInfinite"); + PADDLE_ENFORCE_EQ(found_inf->numel(), 1, + platform::errors::InvalidArgument( + "FoundInfinite must has only one element.")); + const bool* found_inf_data = found_inf->data(); + bool cpu_found_inf_data = false; + if (platform::is_xpu_place(found_inf->place())) { + xpu_memcpy(&cpu_found_inf_data, found_inf_data, sizeof(bool), + XPUMemcpyKind::XPU_DEVICE_TO_HOST); + } else { + cpu_found_inf_data = (*found_inf_data); + } + + for (size_t i = 0; i < xs.size(); ++i) { + auto* out = outs[i]; + T* out_data = out->mutable_data(dev_ctx.GetPlace()); + int num = out->numel(); + if (cpu_found_inf_data) { + VLOG(1) << "-- UpdateLossScaling: Find infinite grads. --"; + int r = 0; + r = xpu::constant(dev_ctx.x_context(), + reinterpret_cast(out_data), num, + XPUTyp(0.0)); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(constant) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + } + } + const bool stop_update = ctx.Attr("stop_update"); + if (stop_update) { + return; + } + + const auto* pre_loss_scaling = ctx.Input("PrevLossScaling"); + const auto* good_in = ctx.Input("InGoodSteps"); + const auto* bad_in = ctx.Input("InBadSteps"); + auto* updated_loss_scaling = ctx.Output("LossScaling"); + auto* good_out = ctx.Output("OutGoodSteps"); + auto* bad_out = ctx.Output("OutBadSteps"); + const MPDType* pre_loss_scaling_data = pre_loss_scaling->data(); + const int* good_in_data = good_in->data(); + const int* bad_in_data = bad_in->data(); + + MPDType* updated_loss_scaling_data = + updated_loss_scaling->mutable_data(dev_ctx.GetPlace()); + int* good_out_data = good_out->mutable_data(dev_ctx.GetPlace()); + int* bad_out_data = bad_out->mutable_data(dev_ctx.GetPlace()); + + const int incr_every_n_steps = ctx.Attr("incr_every_n_steps"); + const int decr_every_n_nan_or_inf = + ctx.Attr("decr_every_n_nan_or_inf"); + const float incr_ratio = ctx.Attr("incr_ratio"); + const float decr_ratio = ctx.Attr("decr_ratio"); + + int cpu_bad_in_data; + int cpu_good_in_data; + MPDType cpu_pre_loss_scaling_data; + if (platform::is_xpu_place(bad_in->place())) { + xpu_memcpy(&cpu_bad_in_data, bad_in_data, sizeof(int), + XPUMemcpyKind::XPU_DEVICE_TO_HOST); + } else { + cpu_bad_in_data = (*bad_in_data); + } + + if (platform::is_xpu_place(good_in->place())) { + xpu_memcpy(&cpu_good_in_data, good_in_data, sizeof(int), + XPUMemcpyKind::XPU_DEVICE_TO_HOST); + } else { + cpu_good_in_data = (*good_in_data); + } + + if (platform::is_xpu_place(pre_loss_scaling->place())) { + xpu_memcpy(&cpu_pre_loss_scaling_data, pre_loss_scaling_data, + sizeof(MPDType), XPUMemcpyKind::XPU_DEVICE_TO_HOST); + } else { + cpu_pre_loss_scaling_data = (*pre_loss_scaling_data); + } + + int cpu_good_out_data = 0; + int cpu_bad_out_data = 0; + MPDType cpu_updated_loss_scaling_data; + + if (cpu_found_inf_data) { + cpu_good_out_data = 0; + cpu_bad_out_data = cpu_bad_in_data + 1; + if (cpu_bad_out_data == decr_every_n_nan_or_inf) { + MPDType new_loss_scaling = cpu_pre_loss_scaling_data * decr_ratio; + cpu_updated_loss_scaling_data = + (new_loss_scaling < static_cast(1)) + ? (static_cast(1)) + : (new_loss_scaling); + cpu_bad_out_data = 0; + } + } else { + cpu_bad_out_data = 0; + cpu_good_out_data = cpu_good_in_data + 1; + if (cpu_good_out_data == incr_every_n_steps) { + MPDType new_loss_scaling = cpu_pre_loss_scaling_data * incr_ratio; + cpu_updated_loss_scaling_data = (std::isfinite(new_loss_scaling)) + ? new_loss_scaling + : cpu_pre_loss_scaling_data; + cpu_good_out_data = 0; + } + } + + // copy to host + memory::Copy(BOOST_GET_CONST(platform::XPUPlace, dev_ctx.GetPlace()), + bad_out_data, platform::CPUPlace(), &cpu_bad_out_data, + sizeof(int)); + memory::Copy(BOOST_GET_CONST(platform::XPUPlace, dev_ctx.GetPlace()), + good_out_data, platform::CPUPlace(), &cpu_good_out_data, + sizeof(int)); + memory::Copy(BOOST_GET_CONST(platform::XPUPlace, dev_ctx.GetPlace()), + updated_loss_scaling_data, platform::CPUPlace(), + &cpu_updated_loss_scaling_data, sizeof(MPDType)); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +namespace plat = paddle::platform; + +REGISTER_OP_XPU_KERNEL(update_loss_scaling, + ops::UpdateLossScalingXPUKernel, + ops::UpdateLossScalingXPUKernel); +#endif diff --git a/paddle/fluid/operators/cast_op_xpu.cc b/paddle/fluid/operators/cast_op_xpu.cc index ca15858cf67d7..c7c0f81f2131f 100644 --- a/paddle/fluid/operators/cast_op_xpu.cc +++ b/paddle/fluid/operators/cast_op_xpu.cc @@ -23,21 +23,9 @@ limitations under the License. */ namespace paddle { namespace operators { -template -class XPUFPTypeTrait { - public: - using Type = T; -}; - -template <> -class XPUFPTypeTrait { - public: - using Type = float16; -}; - template class CastXPUKernel : public framework::OpKernel { - using XPUInTDType = typename XPUFPTypeTrait::Type; + using XPUInTDType = typename XPUTypeTrait::Type; public: void Compute(const framework::ExecutionContext& context) const override { @@ -49,7 +37,6 @@ class CastXPUKernel : public framework::OpKernel { context.Attr("out_dtype")); auto* in_data = in->data(); - // using XPUOutTDType = typename XPUFPTypeTrait::Type; auto numel = in->numel(); auto& dev_ctx = context.template device_context(); int r = -1; diff --git a/paddle/fluid/operators/dropout_op_xpu.cc b/paddle/fluid/operators/dropout_op_xpu.cc index f5d831fa24012..79d239074845a 100644 --- a/paddle/fluid/operators/dropout_op_xpu.cc +++ b/paddle/fluid/operators/dropout_op_xpu.cc @@ -16,11 +16,11 @@ namespace paddle { namespace operators { #ifdef PADDLE_WITH_XPU -static std::map mask_data_tables; -static const int max_data_size = 32 * 1024 * 1024; -static std::mutex s_mask_data_table_lock; + template class DropoutXPUKernel : public framework::OpKernel { + using XPUTyp = typename XPUTypeTrait::Type; + public: void Compute(const framework::ExecutionContext& context) const override { auto* x = context.Input("X"); @@ -30,93 +30,70 @@ class DropoutXPUKernel : public framework::OpKernel { float dropout_prob = context.Attr("dropout_prob"); auto dropout_implementation = context.Attr("dropout_implementation"); - float* mask_data_table = nullptr; + auto& dev_ctx = context.template device_context(); + PADDLE_ENFORCE_EQ(!context.HasInput("Seed"), true, platform::errors::InvalidArgument( ("Input(Seed) not supported on XPU"))); + int is_upscale = (dropout_implementation == "upscale_in_train"); + if (!context.Attr("is_test")) { - int dev_id = - BOOST_GET_CONST(platform::XPUPlace, context.GetPlace()).GetDeviceId(); - int prop = static_cast(dropout_prob * 100); - int is_upscale = (dropout_implementation == "upscale_in_train"); - /* mask_data_tables key contains 3 part: - * | 31-16 | 15-8 | 7-0 | - * | dev_id | prob | is_upscale | - */ - int index = (dev_id << 16) + (prop << 8) + is_upscale; - std::lock_guard lock(s_mask_data_table_lock); - if (mask_data_tables.find(index) == mask_data_tables.end()) { - float* mask_data_host = new float[max_data_size]; - std::random_device rnd; - std::minstd_rand engine; - int seed = - context.Attr("fix_seed") ? context.Attr("seed") : rnd(); - engine.seed(seed); - std::uniform_real_distribution dist(0, 1); - for (size_t i = 0; i < max_data_size; ++i) { - if (dist(engine) < dropout_prob) { - mask_data_host[i] = 0.0f; - } else { - if (is_upscale) { - mask_data_host[i] = 1.0f / static_cast(1.0f - dropout_prob); - } else { - mask_data_host[i] = 1.0; - } - } - } - PADDLE_ENFORCE_EQ( - xpu_malloc(reinterpret_cast(&mask_data_table), - max_data_size * sizeof(float)), - XPU_SUCCESS, - platform::errors::ResourceExhausted( - "\n\nOut of memory error on XPU, Cannot" - "allocate %s memory on XPU. \n\nPlease " - "check whether there is any other process " - "using XPU.\n", - string::HumanReadableSize(max_data_size * sizeof(void*)))); - memory::Copy(BOOST_GET_CONST(platform::XPUPlace, context.GetPlace()), - mask_data_table, platform::CPUPlace(), mask_data_host, - max_data_size * sizeof(float)); - mask_data_tables[index] = mask_data_table; - free(mask_data_host); + std::random_device rnd; + // int seed = (context.Attr("fix_seed")) ? + // int(context.Attr("seed")) : (rnd()); + int seed = 0; + if (context.Attr("fix_seed") == true) { + seed = static_cast(context.Attr("seed")); } else { - mask_data_table = mask_data_tables[index]; + seed = rnd(); } - } - if (!context.Attr("is_test")) { // Train + auto* mask = context.Output("Mask"); auto* mask_data = mask->mutable_data(context.GetPlace()); - size_t size = framework::product(mask->dims()); - auto& dev_ctx = context.template device_context(); - int r = xpu::dropout(dev_ctx.x_context(), mask_data_table, x_data, - mask_data, y_data, max_data_size, size); - PADDLE_ENFORCE_EQ( - r, xpu::Error_t::SUCCESS, - platform::errors::External( - "XPU dropout return wrong value[%d], please check whether " - "Baidu Kunlun Card is properly installed.", - r)); - } else { // Infer - float scale = 0.0f; - if (dropout_implementation == "upscale_in_train") { - scale = 1.0f; - } else { - scale = static_cast(1.0f - dropout_prob); + // Special case when dropout_prob is 1.0 + if (dropout_prob == 1.0f) { + int r = xpu::constant(dev_ctx.x_context(), + reinterpret_cast(y_data), y->numel(), + XPUTyp(0)); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(constant) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + r = xpu::constant(dev_ctx.x_context(), + reinterpret_cast(mask_data), mask->numel(), + XPUTyp(0)); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(constant) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + return; } - auto& dev_ctx = context.template device_context(); - int r = xpu::scale(dev_ctx.x_context(), x->numel(), scale, 0.0f, 0, - x_data, y_data); - PADDLE_ENFORCE_EQ( - r, xpu::Error_t::SUCCESS, - platform::errors::External( - "XPU dropout return wrong value[%d], please check whether " - "Baidu Kunlun Card is properly installed.", - r)); + int r = xpu::dropout(dev_ctx.x_context(), + reinterpret_cast(x->data()), + reinterpret_cast(y->data()), + reinterpret_cast(mask_data), seed, + mask->numel(), is_upscale, dropout_prob); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(dropout) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + } else { + float scale = + (is_upscale) ? (1.0) : (static_cast(1.0f - dropout_prob)); + int r = xpu::scale( + dev_ctx.x_context(), reinterpret_cast(x_data), + reinterpret_cast(y_data), x->numel(), false, scale, 0.0f); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(scale) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); } } }; template class DropoutGradXPUKernel : public framework::OpKernel { + using XPUTyp = typename XPUTypeTrait::Type; + public: void Compute(const framework::ExecutionContext& context) const override { PADDLE_ENFORCE_EQ(!context.Attr("is_test"), true, @@ -127,23 +104,47 @@ class DropoutGradXPUKernel : public framework::OpKernel { auto* mask = context.Input("Mask"); grad_x->mutable_data(context.GetPlace()); auto& dev_ctx = context.template device_context(); - int r = xpu::elementwise_mul(dev_ctx.x_context(), grad_y->data(), - mask->data(), grad_x->data(), - grad_y->numel()); - PADDLE_ENFORCE_EQ( - r, xpu::Error_t::SUCCESS, - platform::errors::External( - "XPU dropout return wrong value[%d], please check whether " - "Baidu Kunlun Card is properly installed.", - r)); + auto& dropout_implementation = + context.Attr("dropout_implementation"); + float dropout_prob = context.Attr("dropout_prob"); + const T* mask_data = mask->data(); + framework::Tensor mask_new; + if (dropout_implementation == "upscale_in_train") { + mask_new = context.AllocateTmpTensor( + mask->dims(), dev_ctx); + float scale = + (dropout_prob == 1.0f) ? (1.0f) : (1.0f / (1.0f - dropout_prob)); + int r = xpu::scale(dev_ctx.x_context(), + reinterpret_cast(mask->data()), + reinterpret_cast(mask_new.data()), + mask->numel(), false, scale, 0.0f); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( + "XPU API(scale) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); + mask_data = mask_new.data(); + } + + int r = xpu::mul( + dev_ctx.x_context(), reinterpret_cast(grad_y->data()), + reinterpret_cast(mask_data), + reinterpret_cast(grad_x->data()), grad_y->numel()); + PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, + platform::errors::External("XPU API(mul) return wrong " + "value[%d %s]", + r, XPUAPIErrorMsg[r])); } }; } // namespace operators } // namespace paddle namespace ops = paddle::operators; +namespace plat = paddle::platform; REGISTER_OP_XPU_KERNEL( - dropout, ops::DropoutXPUKernel); + dropout, ops::DropoutXPUKernel, + ops::DropoutXPUKernel); REGISTER_OP_XPU_KERNEL( dropout_grad, - ops::DropoutGradXPUKernel); + ops::DropoutGradXPUKernel, + ops::DropoutGradXPUKernel); #endif diff --git a/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc b/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc index 8b902acebb4c5..2e902bd277b1e 100644 --- a/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc +++ b/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc @@ -122,33 +122,50 @@ class ElementwiseAddGradXPUKernel : public ElemwiseGradKernel { axis)); std::vector x_dims_vec(max_dim, 1); std::vector y_dims_vec(max_dim, 1); + int x_len = 1; + int y_len = 1; if (x_dims.size() == max_dim) { for (int i = 0; i < max_dim; i++) { x_dims_vec[i] = x_dims[i]; + x_len *= x_dims_vec[i]; } } else { for (int i = 0; i < x_dims.size(); i++) { x_dims_vec[i + axis] = x_dims[i]; + x_len *= x_dims_vec[i]; } } if (y_dims.size() == max_dim) { for (int i = 0; i < max_dim; i++) { y_dims_vec[i] = y_dims[i]; + y_len *= y_dims_vec[i]; } } else { for (int i = 0; i < y_dims.size(); i++) { y_dims_vec[i + axis] = y_dims[i]; + y_len *= y_dims_vec[i]; } } const T* dz_data = dz->data(); + framework::Tensor dx_local_tensor; + framework::Tensor dy_local_tensor; + bool need_wait = false; T* dx_data = nullptr; T* dy_data = nullptr; if (dx) { dx_data = dx->mutable_data(ctx.GetPlace()); + } else { + dx_data = + dx_local_tensor.mutable_data(ctx.GetPlace(), x_len * sizeof(T)); + need_wait = true; } if (dy) { dy_data = dy->mutable_data(ctx.GetPlace()); + } else { + dy_data = + dy_local_tensor.mutable_data(ctx.GetPlace(), y_len * sizeof(T)); + need_wait = true; } auto& dev_ctx = @@ -161,6 +178,9 @@ class ElementwiseAddGradXPUKernel : public ElemwiseGradKernel { platform::errors::External( "XPU kernel Elementwise occur error in XPUElementwise error code ", ret, XPUAPIErrorMsg[ret])); + if (need_wait && dev_ctx.x_context()->xpu_stream) { + dev_ctx.Wait(); + } } }; diff --git a/paddle/fluid/operators/matmul_op_xpu.cc b/paddle/fluid/operators/matmul_op_xpu.cc index 6fa96aca4be14..7097b5327d86f 100644 --- a/paddle/fluid/operators/matmul_op_xpu.cc +++ b/paddle/fluid/operators/matmul_op_xpu.cc @@ -102,6 +102,7 @@ template static void MatMulXPUFunction(const Tensor *x, const Tensor *y, Tensor *out, bool trans_x, bool trans_y, const paddle::framework::ExecutionContext &ctx) { + using XPUType = typename XPUTypeTrait::Type; const auto &x_dims = x->dims(); const auto &y_dims = y->dims(); auto &dev_ctx = @@ -162,34 +163,36 @@ static void MatMulXPUFunction(const Tensor *x, const Tensor *y, Tensor *out, int ldout = n; if (batch_size <= 1) { int r = 0; - r = xpu::fc_fusion( - dev_ctx.x_context(), x->data(), y->data(), data_c, m, n, k, - mat_dim_a.trans_, mat_dim_b.trans_, nullptr, nullptr, nullptr, ldx, ldy, - ldout, alpha, 0, nullptr, xpu::Activation_t::LINEAR); + r = xpu::fc_fusion( + dev_ctx.x_context(), reinterpret_cast(x->data()), + reinterpret_cast(y->data()), + reinterpret_cast(data_c), m, n, k, mat_dim_a.trans_, + mat_dim_b.trans_, nullptr, nullptr, nullptr, ldx, ldy, ldout, alpha, 0, + nullptr, xpu::Activation_t::LINEAR); PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( "XPU fc_fusion kernel return wrong value[%d %s]", r, XPUAPIErrorMsg[r])); } else { // batch matmul - int r = xpu::fc_batched( - dev_ctx.x_context(), // Context* ctx, - batch_size, // int batch_size, - mat_dim_a.trans_, // bool x_trans, - mat_dim_b.trans_, // bool w_trans, - m, // int m, - n, // int n, - k, // int k, - alpha, // float alpha, - reinterpret_cast(x->data()), // const TX* x, - mat_dim_a.stride_, // int stride_a, - reinterpret_cast(y->data()), // const TW* w, - mat_dim_b.stride_, // int stride_b, - 0.0, // float beta, - reinterpret_cast(data_c), // TY* y, - m * n, // int stride_c, - nullptr, // const float* x_maxptr, - nullptr); // const float* w_maxptr + int r = xpu::fc_batched( + dev_ctx.x_context(), // Context* ctx, + batch_size, // int batch_size, + mat_dim_a.trans_, // bool x_trans, + mat_dim_b.trans_, // bool w_trans, + m, // int m, + n, // int n, + k, // int k, + alpha, // float alpha, + reinterpret_cast(x->data()), // const TX* x, + mat_dim_a.stride_, // int stride_a, + reinterpret_cast(y->data()), // const TW* w, + mat_dim_b.stride_, // int stride_b, + 0.0, // float beta, + reinterpret_cast(data_c), // TY* y, + m * n, // int stride_c, + nullptr, // const float* x_maxptr, + nullptr); // const float* w_maxptr PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( @@ -210,10 +213,14 @@ class MatMulXPUKernel : public framework::OpKernel { out->mutable_data(context.GetPlace()); bool trans_x = context.Attr("transpose_X"); bool trans_y = context.Attr("transpose_Y"); - if (std::getenv("XPU_PADDLE_MAT_MUL_FCINT32") != nullptr) { - MatMulXPUFunction(x, y, out, trans_x, trans_y, context); - } else { + if (std::is_same::value) { MatMulXPUFunction(x, y, out, trans_x, trans_y, context); + } else { + if (std::getenv("XPU_PADDLE_MAT_MUL_FCINT32") != nullptr) { + MatMulXPUFunction(x, y, out, trans_x, trans_y, context); + } else { + MatMulXPUFunction(x, y, out, trans_x, trans_y, context); + } } } }; @@ -224,6 +231,7 @@ class MatMulXPUKernel : public framework::OpKernel { template static framework::Tensor XPUFoldHeadAndLastDims( const DeviceContext &context, const framework::Tensor &input) { + using XPUType = typename XPUTypeTrait::Type; auto in_dims = input.dims(); if (in_dims.size() != 3) { return input; @@ -236,8 +244,9 @@ static framework::Tensor XPUFoldHeadAndLastDims( static_cast(in_dims[1]), static_cast(in_dims[2])}; std::vector axis_host = {1, 0, 2}; - int r = xpu::transpose(context.x_context(), input.data(), output.data(), - in_shape_host, axis_host); + int r = xpu::transpose( + context.x_context(), reinterpret_cast(input.data()), + reinterpret_cast(output.data()), in_shape_host, axis_host); PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( "XPU transpose kernel return wrong value[%d %s]", r, @@ -280,10 +289,14 @@ class MatMulGradXPUKernel : public framework::OpKernel { const framework::Tensor &b, bool trans_b, framework::Tensor *out) const { out->mutable_data(context.GetPlace()); - if (std::getenv("XPU_PADDLE_MAT_MUL_GRAD_FCINT32") != nullptr) { - MatMulXPUFunction(&a, &b, out, trans_a, trans_b, context); - } else { + if (std::is_same::value) { MatMulXPUFunction(&a, &b, out, trans_a, trans_b, context); + } else { + if (std::getenv("XPU_PADDLE_MAT_MUL_GRAD_FCINT32") != nullptr) { + MatMulXPUFunction(&a, &b, out, trans_a, trans_b, context); + } else { + MatMulXPUFunction(&a, &b, out, trans_a, trans_b, context); + } } } @@ -370,10 +383,14 @@ class MatMulGradXPUKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; +namespace plat = paddle::platform; REGISTER_OP_XPU_KERNEL( - matmul, ops::MatMulXPUKernel); + matmul, ops::MatMulXPUKernel, + ops::MatMulXPUKernel); REGISTER_OP_XPU_KERNEL( matmul_grad, - ops::MatMulGradXPUKernel); + ops::MatMulGradXPUKernel, + ops::MatMulGradXPUKernel); #endif diff --git a/paddle/fluid/operators/matmul_v2_op_xpu.cc b/paddle/fluid/operators/matmul_v2_op_xpu.cc index d992ef847db2a..ae1e9358f6811 100644 --- a/paddle/fluid/operators/matmul_v2_op_xpu.cc +++ b/paddle/fluid/operators/matmul_v2_op_xpu.cc @@ -25,6 +25,7 @@ template static void MatMulXPUFunction(const Tensor* x, const Tensor* y, Tensor* out, bool trans_x, bool trans_y, const paddle::framework::ExecutionContext& ctx) { + using XPUType = typename XPUTypeTrait::Type; const auto& x_dims = x->dims(); const auto& y_dims = y->dims(); auto& dev_ctx = @@ -75,9 +76,11 @@ static void MatMulXPUFunction(const Tensor* x, const Tensor* y, Tensor* out, int batch_size = mat_dim_a.batch_size_; if (batch_size <= 1) { int r = 0; - r = xpu::fc(dev_ctx.x_context(), x->data(), y->data(), - data_c, m, n, k, mat_dim_a.trans_, - mat_dim_b.trans_, nullptr, nullptr, nullptr); + r = xpu::fc( + dev_ctx.x_context(), reinterpret_cast(x->data()), + reinterpret_cast(y->data()), + reinterpret_cast(data_c), m, n, k, mat_dim_a.trans_, + mat_dim_b.trans_, nullptr, nullptr, nullptr); PADDLE_ENFORCE_EQ( r, XPU_SUCCESS, platform::errors::External( @@ -87,24 +90,24 @@ static void MatMulXPUFunction(const Tensor* x, const Tensor* y, Tensor* out, r, XPUAPIErrorMsg[r], m, n, k, mat_dim_a.trans_, mat_dim_b.trans_)); } else { // batch matmul - int r = xpu::fc_batched( - dev_ctx.x_context(), // Context* ctx, - batch_size, // int batch_size, - mat_dim_a.trans_, // bool x_trans, - mat_dim_b.trans_, // bool w_trans, - m, // int m, - n, // int n, - k, // int k, - 1.0, // float alpha, - reinterpret_cast(x->data()), // const TX* x, - mat_dim_a.stride_, // int stride_a, - reinterpret_cast(y->data()), // const TW* w, - mat_dim_b.stride_, // int stride_b, - 0.0, // float beta, - reinterpret_cast(data_c), // TY* y, - m * n, // int stride_c, - nullptr, // const float* x_maxptr, - nullptr); // const float* w_maxptr + int r = xpu::fc_batched( + dev_ctx.x_context(), // Context* ctx, + batch_size, // int batch_size, + mat_dim_a.trans_, // bool x_trans, + mat_dim_b.trans_, // bool w_trans, + m, // int m, + n, // int n, + k, // int k, + 1.0, // float alpha, + reinterpret_cast(x->data()), // const TX* x, + mat_dim_a.stride_, // int stride_a, + reinterpret_cast(y->data()), // const TW* w, + mat_dim_b.stride_, // int stride_b, + 0.0, // float beta, + reinterpret_cast(data_c), // TY* y, + m * n, // int stride_c, + nullptr, // const float* x_maxptr, + nullptr); // const float* w_maxptr PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( @@ -123,10 +126,14 @@ class MatMulV2XPUKernel : public framework::OpKernel { bool trans_x = ctx.Attr("trans_x"); bool trans_y = ctx.Attr("trans_y"); out->mutable_data(ctx.GetPlace()); - if (std::getenv("XPU_PADDLE_MAT_MUL_V2_FCINT32") != nullptr) { - MatMulXPUFunction(x, y, out, trans_x, trans_y, ctx); - } else { + if (std::is_same::value) { MatMulXPUFunction(x, y, out, trans_x, trans_y, ctx); + } else { + if (std::getenv("XPU_PADDLE_MAT_MUL_V2_FCINT32") != nullptr) { + MatMulXPUFunction(x, y, out, trans_x, trans_y, ctx); + } else { + MatMulXPUFunction(x, y, out, trans_x, trans_y, ctx); + } } } }; @@ -134,6 +141,7 @@ class MatMulV2XPUKernel : public framework::OpKernel { template static framework::Tensor XPUFoldHeadAndLastDims( const DeviceContext& context, const framework::Tensor& input) { + using XPUType = typename XPUTypeTrait::Type; auto in_dims = input.dims(); if (in_dims.size() != 3) { return input; @@ -147,8 +155,9 @@ static framework::Tensor XPUFoldHeadAndLastDims( static_cast(in_dims[2])}; std::vector axis_host = {1, 0, 2}; - int r = xpu::transpose(context.x_context(), input.data(), output.data(), - in_shape_host, axis_host); + int r = xpu::transpose( + context.x_context(), reinterpret_cast(input.data()), + reinterpret_cast(output.data()), in_shape_host, axis_host); PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External( "XPU transpose kernel return wrong value[%d %s]", r, @@ -166,10 +175,14 @@ class MatMulV2XPUGradKernel : public framework::OpKernel { const framework::Tensor& b, bool trans_b, framework::Tensor* out) const { out->mutable_data(ctx.GetPlace()); - if (std::getenv("XPU_PADDLE_MAT_MUL_GRAD_V2_FCINT32") != nullptr) { - MatMulXPUFunction(&a, &b, out, trans_a, trans_b, ctx); - } else { + if (std::is_same::value) { MatMulXPUFunction(&a, &b, out, trans_a, trans_b, ctx); + } else { + if (std::getenv("XPU_PADDLE_MAT_MUL_GRAD_V2_FCINT32") != nullptr) { + MatMulXPUFunction(&a, &b, out, trans_a, trans_b, ctx); + } else { + MatMulXPUFunction(&a, &b, out, trans_a, trans_b, ctx); + } } } @@ -261,8 +274,10 @@ class MatMulV2XPUGradKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; - -REGISTER_OP_XPU_KERNEL(matmul_v2, ops::MatMulV2XPUKernel); -REGISTER_OP_XPU_KERNEL(matmul_v2_grad, ops::MatMulV2XPUGradKernel); +namespace plat = paddle::platform; +REGISTER_OP_XPU_KERNEL(matmul_v2, ops::MatMulV2XPUKernel, + ops::MatMulV2XPUKernel); +REGISTER_OP_XPU_KERNEL(matmul_v2_grad, ops::MatMulV2XPUGradKernel, + ops::MatMulV2XPUGradKernel); #endif diff --git a/paddle/fluid/operators/softmax_op_xpu.cc b/paddle/fluid/operators/softmax_op_xpu.cc index ed7034ef6ab41..3527478f76610 100644 --- a/paddle/fluid/operators/softmax_op_xpu.cc +++ b/paddle/fluid/operators/softmax_op_xpu.cc @@ -47,8 +47,8 @@ class SoftmaxXPUKernel : public framework::OpKernel { int len = x->numel(); T* clip_x_data = clip_x.mutable_data(context.GetPlace(), len * sizeof(T)); - r = xpu::clip(dev_ctx.x_context(), x->data(), clip_x_data, len, - -1e30, 1e30); + r = xpu::clip_v2(dev_ctx.x_context(), x->data(), clip_x_data, len, + static_cast(-1e20), static_cast(1e20)); PADDLE_ENFORCE_EQ(r, XPU_SUCCESS, platform::errors::External("XPU API(clip) return wrong " "value[%d %s]", diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op_xpu.cc b/paddle/fluid/operators/softmax_with_cross_entropy_op_xpu.cc index 8635def2ecf13..a79e31eb8d028 100644 --- a/paddle/fluid/operators/softmax_with_cross_entropy_op_xpu.cc +++ b/paddle/fluid/operators/softmax_with_cross_entropy_op_xpu.cc @@ -54,8 +54,9 @@ class SoftmaxWithCrossEntropyXPUKernel : public framework::OpKernel { int len = logits->numel(); T* clip_logits_data = clip_logits.mutable_data(context.GetPlace(), len * sizeof(T)); - r = xpu::clip(dev_ctx.x_context(), logits->data(), clip_logits_data, - len, -1e30, 1e30); + r = xpu::clip_v2(dev_ctx.x_context(), logits->data(), + clip_logits_data, len, static_cast(-1e20), + static_cast(1e20)); PADDLE_ENFORCE_EQ( r, xpu::Error_t::SUCCESS, platform::errors::External("XPU kernel error. clip " diff --git a/paddle/fluid/platform/xpu_header.h b/paddle/fluid/platform/xpu_header.h index 9f2befc123f22..99f4224b5d408 100644 --- a/paddle/fluid/platform/xpu_header.h +++ b/paddle/fluid/platform/xpu_header.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include #include "paddle/fluid/platform/errors.h" +#include "paddle/fluid/platform/float16.h" #include "xpu/api.h" #include "xpu/refactor/fusion.h" #include "xpu/refactor/math.h" @@ -58,4 +59,16 @@ static std::map XPUAPIErrorMsg = { {xpu::Error_t::RUNTIME_ERROR, "xpu api runtime error"}, {xpu::Error_t::NO_ENOUGH_WORKSPACE, "xpu api no enough workspace"}}; +template +class XPUTypeTrait { + public: + using Type = T; +}; + +template <> +class XPUTypeTrait { + public: + using Type = float16; +}; + #endif diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 560d8c892b09f..fd4ae63265366 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -224,7 +224,9 @@ OpSupportedInfos(const std::string &place, [](unsigned char c) { return std::toupper(c); }); using fn_type = std::add_pointer::type; std::unordered_map is_target_place{ - {"GPU", &platform::is_gpu_place}, {"CPU", &platform::is_cpu_place}, + {"GPU", &platform::is_gpu_place}, + {"CPU", &platform::is_cpu_place}, + {"XPU", &platform::is_xpu_place}, }; PADDLE_ENFORCE_NE( is_target_place.count(query_place), 0, diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py index f940f6a3143a0..7c6f32e1e8e62 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py @@ -149,8 +149,14 @@ def _update_list(self): # The set of ops that don't support fp16 calculation # lookup_table fp16 is slower than fp32, though fp16 is supported. -_, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'GPU', core.VarDesc.VarType.FP16) +_sys_unsupported_fp16_list = [] +if core.is_compiled_with_xpu(): + _, _, _sys_unsupported_fp16_list = core.op_supported_infos( + 'XPU', core.VarDesc.VarType.FP16) +else: + _, _, _sys_unsupported_fp16_list = core.op_supported_infos( + 'GPU', core.VarDesc.VarType.FP16) + unsupported_fp16_list = {'lookup_table', 'lookup_table_v2'} | _sys_unsupported_fp16_list diff --git a/python/paddle/fluid/dygraph/amp/auto_cast.py b/python/paddle/fluid/dygraph/amp/auto_cast.py index 4ff08337875c0..6121732bf1f72 100644 --- a/python/paddle/fluid/dygraph/amp/auto_cast.py +++ b/python/paddle/fluid/dygraph/amp/auto_cast.py @@ -128,9 +128,10 @@ def amp_guard(enable=True, custom_white_list=None, custom_black_list=None): raise ValueError( "current_tracer is None, maybe it is not in imperative mode.") - if enable and not tracer._expected_place.is_gpu_place(): + if enable and not (tracer._expected_place.is_gpu_place() or + tracer._expected_place.is_xpu_place()): warnings.warn( - 'amp_guard can only be enabled on CUDAPlace, current place is %s, so it makes no effect.' + 'amp_guard can only be enabled on CUDAPlace and XPUPlace, current place is %s, so it makes no effect.' % tracer._expected_place) enable = False diff --git a/python/paddle/fluid/dygraph/amp/loss_scaler.py b/python/paddle/fluid/dygraph/amp/loss_scaler.py index ff57f30dcd2ec..e0bd60fbeb4a7 100644 --- a/python/paddle/fluid/dygraph/amp/loss_scaler.py +++ b/python/paddle/fluid/dygraph/amp/loss_scaler.py @@ -90,9 +90,10 @@ def __init__(self, raise ValueError( "current_tracer is None, maybe it is not in imperative mode.") - if enable and not tracer._expected_place.is_gpu_place(): + if enable and not (tracer._expected_place.is_gpu_place() or + tracer._expected_place.is_xpu_place()): warnings.warn( - 'AmpScaler can only be enabled on CUDAPlace, current place is %s, so it makes no effect.' + 'AmpScaler can only be enabled on CUDAPlace and XPUPlace, current place is %s, so it makes no effect.' % tracer._expected_place) enable = False diff --git a/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py new file mode 100644 index 0000000000000..9a2976f82a460 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py @@ -0,0 +1,99 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append("..") +import paddle +import unittest +import numpy as np +from op_test_xpu import XPUOpTest +from op_test import OpTest, skip_check_grad_ci +import paddle.fluid as fluid +paddle.enable_static() + + +class TestCheckFiniteAndUnscaleOp(XPUOpTest): + def setUp(self): + self.op_type = "check_finite_and_unscale" + self.init_dtype() + x = np.random.random((1024, 1024)).astype(self.dtype) + scale = np.random.random((1)).astype(self.dtype) + # self.attrs = {'stop_gradient': True} + self.inputs = {'X': [('x0', x)], 'Scale': scale} + self.outputs = { + 'FoundInfinite': np.array([0]), + 'Out': [('out0', x / scale)], + } + + def init_dtype(self): + self.dtype = np.float32 + + def test_check_output(self): + if paddle.is_compiled_with_xpu(): + place = paddle.XPUPlace(0) + self.check_output_with_place(place) + + +# class TestCheckFiniteAndUnscaleOpWithNan(XPUOpTest): +# def setUp(self): +# self.op_type = "check_finite_and_unscale" +# self.init_dtype() +# x = np.random.random((1024, 1024)).astype(self.dtype) +# x[128][128] = np.nan +# print("x shape = ", x.shape) +# print(x) +# scale = np.random.random((1)).astype(self.dtype) + +# self.inputs = {'X': [('x0', x)], 'Scale': scale} +# self.outputs = { +# 'FoundInfinite': np.array([1]), +# 'Out': [('out0', x)], +# } + +# def init_dtype(self): +# self.dtype = np.float32 + +# def test_check_output(self): +# # When input contains nan, do not check the output, +# # since the output may be nondeterministic and will be discarded. +# if paddle.is_compiled_with_xpu(): +# place = paddle.XPUPlace(0) +# self.check_output_with_place(place, no_check_set=['Out']) + +# class TestCheckFiniteAndUnscaleOpWithInf(XPUOpTest): +# def setUp(self): +# self.op_type = "check_finite_and_unscale" +# self.init_dtype() +# x = np.random.random((1024, 1024)).astype(self.dtype) +# x[128][128] = np.inf +# scale = np.random.random((1)).astype(self.dtype) + +# self.inputs = {'X': [('x0', x)], 'Scale': scale} +# self.outputs = { +# 'FoundInfinite': np.array([1]), +# 'Out': [('out0', x)], +# } + +# def init_dtype(self): +# self.dtype = np.float32 + +# def test_check_output(self): +# # When input contains inf, do not check the output, +# # since the output may be nondeterministic and will be discarded. +# if paddle.is_compiled_with_xpu(): +# place = paddle.XPUPlace(0) +# self.check_output_with_place(place, no_check_set=['Out']) + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py index 6c3368c3b6bfc..ca3b3a418abf6 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py @@ -22,9 +22,11 @@ import paddle import paddle.fluid as fluid from paddle.fluid import Program, program_guard +from op_test_xpu import XPUOpTest +paddle.enable_static() -class TestDropoutOp(OpTest): +class TestDropoutOp(XPUOpTest): def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} @@ -47,7 +49,7 @@ def test_check_grad_normal(self): self.check_grad_with_place(place, ['X'], 'Out') -class TestDropoutOpInput1d(OpTest): +class TestDropoutOpInput1d(XPUOpTest): def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((2000, )).astype("float32")} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py new file mode 100644 index 0000000000000..33b13081b5442 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py @@ -0,0 +1,245 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import sys +sys.path.append("..") +import numpy as np +from op_test import OpTest +from op_test_xpu import XPUOpTest +import paddle +import paddle.fluid as fluid +import paddle.fluid.contrib.mixed_precision.amp_nn as amp_nn + +paddle.enable_static() + + +class TestUpdateLossScalingOp(XPUOpTest): + def setUp(self): + self.op_type = "update_loss_scaling" + self.init() + found_inf = np.array([False], dtype=np.bool) + x = np.random.random((1024, 1024)).astype(self.dtype) + + self.inputs = { + 'X': [('x0', x)], + 'FoundInfinite': found_inf, + 'PrevLossScaling': self.prev_loss_scaling, + 'InGoodSteps': self.num_good_steps, + 'InBadSteps': self.num_bad_steps + } + + self.outputs = { + 'Out': [('out0', x)], + 'LossScaling': self.prev_loss_scaling * self.incr_ratio, + 'OutGoodSteps': self.zero_steps, + 'OutBadSteps': self.zero_steps + } + + def init(self): + self.incr_ratio = 2.0 + self.decr_ratio = 0.8 + self.dtype = np.float32 + self.prev_loss_scaling = np.array([2048]).astype(self.dtype) + self.num_good_steps = np.array([999], dtype=np.int32) + self.num_bad_steps = np.array([1], dtype=np.int32) + self.zero_steps = np.array([0], dtype=np.int32) + self.attrs = { + 'incr_every_n_steps': 1000, + 'decr_every_n_nan_or_inf': 2, + 'incr_ratio': self.incr_ratio, + 'decr_ratio': self.decr_ratio, + } + + def test_check_output(self): + if paddle.is_compiled_with_xpu(): + place = paddle.XPUPlace(0) + self.check_output_with_place(place, no_check_set=['Out']) + + +class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): + def setUp(self): + self.op_type = "update_loss_scaling" + self.init() + found_inf = np.array([True], dtype=np.bool) + x = np.random.random((1024, 1024)).astype(self.dtype) + i = np.random.randint(0, 1024, 1) + j = np.random.randint(0, 1024, 1) + x[i[0]][j[0]] = np.inf + + self.inputs = { + 'X': [('x0', x)], + 'FoundInfinite': found_inf, + 'PrevLossScaling': self.prev_loss_scaling, + 'InGoodSteps': self.num_good_steps, + 'InBadSteps': self.num_bad_steps + } + + self.outputs = { + 'Out': [('out0', np.zeros_like(x))], + 'LossScaling': self.prev_loss_scaling * self.decr_ratio, + 'OutGoodSteps': self.zero_steps, + 'OutBadSteps': self.zero_steps + } + + def test_check_output(self): + if paddle.is_compiled_with_xpu(): + place = paddle.XPUPlace(0) + self.check_output_with_place(place) + #self.check_output() + + +class TestUpdateLossScalingLayer(unittest.TestCase): + def loss_scaling_check(self, scope=fluid.Scope()): + a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') + b = fluid.data(name="b", shape=[512, 128], dtype='float32') + x = [a, b] + found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32') + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32') + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32') + + a_v = np.random.random([1024, 1024]).astype('float32') + b_v = np.random.random([512, 128]).astype('float32') + found_inf_v = np.array([False]).astype('bool') + prev_loss_scaling_v = np.array([2048]).astype('float32') + num_good_steps_v = np.array([999], dtype=np.int32) + num_bad_steps_v = np.array([1], dtype=np.int32) + + incr_every_n_steps = 1000 + decr_every_n_nan_or_inf = 2 + incr_ratio = 2 + decr_ratio = 0.8 + + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling") + + place = fluid.XPUPlace(0) + exe = fluid.Executor(place) + with fluid.scope_guard(scope): + exe.run(fluid.default_startup_program()) + result_v = exe.run(feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v + }, + fetch_list=[ + result, x, found_inf, prev_loss_scaling, + num_good_steps, num_bad_steps + ]) + assert np.array_equal(result_v[0], a_v) + assert np.array_equal(result_v[1], b_v) + assert np.array_equal(result_v[0], result_v[2]) + assert np.array_equal(result_v[1], result_v[3]) + assert np.array_equal(result_v[4], found_inf_v) + assert np.array_equal(result_v[5], prev_loss_scaling_v * incr_ratio) + assert np.array_equal(result_v[6], np.zeros_like(num_good_steps_v)) + assert np.array_equal(result_v[7], np.zeros_like(num_bad_steps_v)) + + def loss_scaling_check_inf(self, use_cuda=True, scope=fluid.Scope()): + a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') + b = fluid.data(name="b", shape=[512, 128], dtype='float32') + x = [a, b] + found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32') + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32') + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32') + + a_v = np.random.random([1024, 1024]).astype('float32') + b_v = np.random.random([512, 128]).astype('float32') + i = np.random.randint(0, 1024, 1) + j = np.random.randint(0, 1024, 1) + a_v[i[0]][j[0]] = np.inf + found_inf_v = np.array([True]).astype('bool') + prev_loss_scaling_v = np.array([2048]).astype('float32') + num_good_steps_v = np.array([999], dtype=np.int32) + num_bad_steps_v = np.array([1], dtype=np.int32) + + incr_every_n_steps = 1000 + decr_every_n_nan_or_inf = 2 + incr_ratio = 2 + decr_ratio = 0.8 + + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling") + + place = fluid.XPUPlace(0) + exe = fluid.Executor(place) + with fluid.scope_guard(scope): + exe.run(fluid.default_startup_program()) + result_v = exe.run(feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v + }, + fetch_list=[ + result, x, found_inf, prev_loss_scaling, + num_good_steps, num_bad_steps + ]) + assert np.array_equal(result_v[0], np.zeros_like(a_v)) + assert np.array_equal(result_v[1], np.zeros_like(b_v)) + assert np.array_equal(result_v[2], np.zeros_like(a_v)) + assert np.array_equal(result_v[3], np.zeros_like(b_v)) + assert np.array_equal(result_v[4], found_inf_v) + assert np.array_equal(result_v[5], prev_loss_scaling_v * decr_ratio) + assert np.array_equal(result_v[6], np.zeros_like(num_good_steps_v)) + assert np.array_equal(result_v[7], np.zeros_like(num_bad_steps_v)) + + def test_loss_scaling(self): + main = fluid.Program() + startup = fluid.Program() + with fluid.unique_name.guard(): + with fluid.program_guard(main, startup): + self.loss_scaling_check() + + def test_loss_scaling_inf(self): + main = fluid.Program() + startup = fluid.Program() + with fluid.unique_name.guard(): + with fluid.program_guard(main, startup): + self.loss_scaling_check_inf() + + +if __name__ == '__main__': + unittest.main() From 999c29173093f7ca494d7bc27a30d0a74f9baa39 Mon Sep 17 00:00:00 2001 From: WeiXin Date: Mon, 12 Jul 2021 17:35:36 +0800 Subject: [PATCH 148/156] [Cherry-pick]Delete the function of saving layer object. (#34039) * Save all the information of 'ParamBase' in 'Layer'. (#33500) * Save all the information of 'ParamBase' in 'Layer'. * edit unittest * delete the function of saving layer object. (#33697) * delete the function of saving layer object. * edit doc of paddle.save/load and polish error message --- .../tests/unittests/test_paddle_save_load.py | 14 ++------ python/paddle/framework/io.py | 32 ++++++++++++++----- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index be2a6a653cc6f..af8718a2121b1 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -869,21 +869,11 @@ def test_save_load_layer(self): layer2 = LinearNet() layer1.eval() layer2.eval() + origin_layer = (layer1, layer2) origin = (layer1(inps), layer2(inps)) path = "test_save_load_layer_/layer.pdmodel" - paddle.save((layer1, layer2), path) - - # static - paddle.enable_static() with self.assertRaises(ValueError): - paddle.load(path) - # dygraph - paddle.disable_static() - - loaded_layer = paddle.load(path) - loaded_result = [l(inps) for l in loaded_layer] - for i in range(len(origin)): - self.assertTrue((origin[i] - loaded_result[i]).abs().max() < 1e-10) + paddle.save(origin_layer, path) if __name__ == '__main__': diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 1705db50d391a..01145e8563cf3 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -232,7 +232,7 @@ def _pickle_save(obj, f, protocol): raise ValueError("Expected 1<'protocol'<5, but received protocol={}". format(protocol)) - def reudce_varbase(self): + def reduce_varbase(self): data = self.numpy() name = self.name @@ -243,16 +243,32 @@ def reduce_LoDTensor(self): return (eval, ('data', {'data': data})) + def reduce_Layer(self): + raise ValueError( + "paddle do not support saving `paddle.nn.Layer` object.") + + dispatch_table_layer = dict() + + def create_layer_dispatch_table(layer): + dispatch_table_layer[layer.__class__] = reduce_Layer + return layer + + _parse_every_object(obj, lambda v: isinstance(v, core.Layer), + create_layer_dispatch_table) + def add_dispatch_table(): # This is not a good method, because the pickle module has been modified. - pickle.dispatch_table[core.VarBase] = reudce_varbase - pickle.dispatch_table[ParamBase] = reudce_varbase + pickle.dispatch_table[core.VarBase] = reduce_varbase + pickle.dispatch_table[ParamBase] = reduce_varbase pickle.dispatch_table[core.LoDTensor] = reduce_LoDTensor + pickle.dispatch_table.update(dispatch_table_layer) def pop_dispatch_table(): pickle.dispatch_table.pop(core.VarBase) pickle.dispatch_table.pop(core.LoDTensor) pickle.dispatch_table.pop(ParamBase) + for k in dispatch_table_layer: + pickle.dispatch_table.pop(k) # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' if sys.platform == 'darwin' and sys.version_info.major == 3: @@ -272,10 +288,10 @@ def pop_dispatch_table(): pickler = pickle.Pickler(f, protocol) pickler.dispatch_table = copyreg.dispatch_table.copy() - pickler.dispatch_table[core.VarBase] = reudce_varbase + pickler.dispatch_table[core.VarBase] = reduce_varbase pickler.dispatch_table[core.LoDTensor] = reduce_LoDTensor - pickler.dispatch_table[ParamBase] = reudce_varbase - + pickler.dispatch_table[ParamBase] = reduce_varbase + pickler.dispatch_table.update(dispatch_table_layer) pickler.dump(obj) @@ -496,7 +512,7 @@ def save(obj, path, protocol=4, **configs): Save an object to the specified path. .. note:: - Now supports saving ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor, Program. + Now supports saving ``state_dict`` of Layer/Optimizer, Tensor and nested structure containing Tensor, Program. .. note:: Different from ``paddle.jit.save``, since the save result of ``paddle.save`` is a single file, @@ -690,7 +706,7 @@ def load(path, **configs): Load an object can be used in paddle from specified path. .. note:: - Now supports loading ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor, Program. + Now supports loading ``state_dict`` of Layer/Optimizer, Tensor and nested structure containing Tensor, Program. .. note:: In order to use the model parameters saved by paddle more efficiently, From 1d1ca0f877e19f5925d35e4ee94e1c27919459af Mon Sep 17 00:00:00 2001 From: WeiXin Date: Thu, 15 Jul 2021 16:47:10 +0800 Subject: [PATCH 149/156] [Cherry-Pick]Support finetuning the model saved on the MAC on the Linux (#34027) (#34154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复《jit.save在Mac系统上保存的模型，在Linux平台上无法对模型进行重训练》的问题。 原始PR： #34027 --- paddle/fluid/operators/matmul_op.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/operators/matmul_op.cc b/paddle/fluid/operators/matmul_op.cc index c12aecc9ba516..fdd11486270cd 100644 --- a/paddle/fluid/operators/matmul_op.cc +++ b/paddle/fluid/operators/matmul_op.cc @@ -232,7 +232,9 @@ class MatMulGradKernel : public framework::OpKernel { int head_number = 1; #if defined(PADDLE_WITH_MKLML) && !defined(PADDLE_WITH_CUDA) && \ !defined(PADDLE_WITH_HIP) - head_number = context.Attr("head_number"); + if (context.HasAttr("head_number")) { + head_number = context.Attr("head_number"); + } #endif if (head_number <= 1 && a.dims().size() == 3 && b.dims().size() <= 2) { From a456a1be05f3f4c6fcc4a888bb1ba87f7e07b762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=98=8E=E5=86=AC?= <78149749+winter-wang@users.noreply.github.com> Date: Mon, 19 Jul 2021 10:39:54 +0800 Subject: [PATCH 150/156] add the size of libpaddle_inference.so to Inference CI, test=develop (#34063) (#34168) --- paddle/scripts/paddle_build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 2af767472face..66420a15064fc 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -424,8 +424,11 @@ EOF cp -r paddle_inference_install_dir paddle_inference tar -czf paddle_inference.tgz paddle_inference buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/paddle_inference.tgz |awk '{print $1}') + soLibSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/paddle_inference_install_dir/paddle/lib/libpaddle_inference.so |awk '{print $1}') echo "Paddle_Inference Size: $buildSize" + echo "Paddle_Inference Dynamic Library Size: $soLibSize" echo "ipipe_log_param_Paddle_Inference_Size: $buildSize" >> ${PADDLE_ROOT}/build/build_summary.txt + echo "ipipe_log_param_Paddle_Inference_So_Size: $soLibSize" >> ${PADDLE_ROOT}/build/build_summary.txt elif [ "$1" == "paddle_inference_c" ]; then cd ${PADDLE_ROOT}/build cp -r paddle_inference_c_install_dir paddle_inference_c From 519df32e226e8742c9bbc7744f8c5b2cd3818c0b Mon Sep 17 00:00:00 2001 From: Wilber Date: Mon, 19 Jul 2021 15:23:03 +0800 Subject: [PATCH 151/156] cherry-pick 34040 (#34228) --- .../inference/tensorrt/plugin/anchor_generator_op_plugin.cu | 4 ++-- paddle/fluid/inference/tensorrt/plugin/roi_align_op_plugin.cu | 2 +- paddle/fluid/inference/tensorrt/plugin/yolo_box_op_plugin.cu | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/plugin/anchor_generator_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/anchor_generator_op_plugin.cu index 01ee86ceb48a9..93cb1c29ff2a6 100644 --- a/paddle/fluid/inference/tensorrt/plugin/anchor_generator_op_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/anchor_generator_op_plugin.cu @@ -215,7 +215,7 @@ const char* AnchorGeneratorPlugin::getPluginNamespace() const { nvinfer1::DataType AnchorGeneratorPlugin::getOutputDataType( int index, const nvinfer1::DataType* input_type, int nb_inputs) const { - return data_type_; + return input_type[0]; } bool AnchorGeneratorPlugin::isOutputBroadcastAcrossBatch( @@ -456,7 +456,7 @@ int AnchorGeneratorPluginDynamic::enqueue( nvinfer1::DataType AnchorGeneratorPluginDynamic::getOutputDataType( int index, const nvinfer1::DataType* inputTypes, int nbInputs) const { - return data_type_; + return inputTypes[0]; } const char* AnchorGeneratorPluginDynamic::getPluginType() const { diff --git a/paddle/fluid/inference/tensorrt/plugin/roi_align_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/roi_align_op_plugin.cu index 6e7ed0054f502..61e9144b9c8d4 100644 --- a/paddle/fluid/inference/tensorrt/plugin/roi_align_op_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/roi_align_op_plugin.cu @@ -304,7 +304,7 @@ int RoiAlignPluginDynamic::enqueue(const nvinfer1::PluginTensorDesc* inputDesc, nvinfer1::DataType RoiAlignPluginDynamic::getOutputDataType( int index, const nvinfer1::DataType* inputTypes, int nbInputs) const { - return data_type_; + return inputTypes[0]; } const char* RoiAlignPluginDynamic::getPluginType() const { diff --git a/paddle/fluid/inference/tensorrt/plugin/yolo_box_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/yolo_box_op_plugin.cu index 13d07e774036a..fb2712e823a85 100644 --- a/paddle/fluid/inference/tensorrt/plugin/yolo_box_op_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/yolo_box_op_plugin.cu @@ -295,7 +295,7 @@ const char* YoloBoxPlugin::getPluginNamespace() const { nvinfer1::DataType YoloBoxPlugin::getOutputDataType( int index, const nvinfer1::DataType* input_type, int nb_inputs) const { - return data_type_; + return input_type[0]; } bool YoloBoxPlugin::isOutputBroadcastAcrossBatch(int output_index, From 8db945a90560c2c4b550e7caa97333cdd2ebde3e Mon Sep 17 00:00:00 2001 From: Chen Long <1300851984@qq.com> Date: Mon, 19 Jul 2021 20:18:24 +0800 Subject: [PATCH 152/156] Update while loop (#34229) * update readme test=document_fix * update while loop docs test=document_fix --- python/paddle/fluid/layers/control_flow.py | 32 ++++++---------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 3a06b84d111c4..fff65f9f46e7b 100755 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1133,30 +1133,14 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): refer to :ref:`api_guide_Name`. Default is None. Returns: - A list or tuple of tensors or LoDTensorArrays which returned by ``body`` . - - Returen type: - list(Variable)|tuple(Variable). - - Raises: - TypeError: If the type of ``cond`` is not callable. - TypeError: If the type of ``body`` is not callable. - TypeError: If the type of ``loop_vars`` is not list or tuple. - TypeError: If the type of ``cond`` returns is not Variable. - TypeError: If the type of ``cond`` returns is not a boolean variable. - TypeError: If the shape of ``cond`` returns is not equals 1. - ValueError: If the ``var_loops`` is empty. - ValueError: If the length or type of ``body`` returns is not same as ``loop_vars``. + A list or tuple of Tensors or LoDTensorArrays which returned by ``body`` . Examples: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers import paddle paddle.enable_static() - def cond(i, ten): return i < ten @@ -1164,14 +1148,14 @@ def body(i, ten): i = i + 1 return [i, ten] - main_program = fluid.default_main_program() - startup_program = fluid.default_startup_program() - with fluid.program_guard(main_program, startup_program): - i = layers.fill_constant(shape=[1], dtype='int64', value=0) # loop counter - ten = layers.fill_constant(shape=[1], dtype='int64', value=10) # loop length - i, ten = layers.while_loop(cond, body, [i, ten]) + main_program = paddle.static.default_main_program() + startup_program = paddle.static.default_startup_program() + with paddle.static.program_guard(main_program, startup_program): + i = paddle.full(shape=[1], fill_value=0, dtype='int64') # loop counter + ten = paddle.full(shape=[1], fill_value=10, dtype='int64') # loop length + i, ten = paddle.static.nn.while_loop(cond, body, [i, ten]) - exe = fluid.Executor(fluid.CPUPlace()) + exe = paddle.static.Executor(paddle.CPUPlace()) res = exe.run(main_program, feed={}, fetch_list=[i]) print(res) # [array([10])] """ From 4ffd33958edacf9cac8695c8073ff42aa59b2350 Mon Sep 17 00:00:00 2001 From: 0x45f <23097963+0x45f@users.noreply.github.com> Date: Wed, 21 Jul 2021 10:13:48 +0800 Subject: [PATCH 153/156] [Cherry-pick][Dy2Stat]Support Nest sequtial container (#34246) #34262 * support Nest sequtial container * rename model path --- .../dygraph_to_static/convert_call_func.py | 2 +- .../dygraph_to_static/test_container.py | 38 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py index a621f68c6545a..b62c16989fbe7 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py @@ -88,7 +88,7 @@ def is_unsupported(func): for v in m.__dict__.values(): func_in_dict = func == v if isinstance(func_in_dict, (list, numpy.ndarray)): - func_in_dict = any(func_in_dict) + func_in_dict = numpy.array(func_in_dict).any() if func_in_dict: translator_logger.log( 2, diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py index 647c9e9672cf0..2c82f5c699087 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py @@ -47,10 +47,30 @@ def forward(self, x): return out +class NestSequentialNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + group1 = paddle.nn.Sequential( + paddle.nn.Linear(10, 10), + paddle.nn.Sigmoid(), ) + group2 = paddle.nn.Sequential( + paddle.nn.Linear(10, 3), + paddle.nn.ReLU(), ) + self.layers = paddle.nn.Sequential(group1, group2) + + def forward(self, x): + return self.layers(x) + + class TestSequential(unittest.TestCase): def setUp(self): paddle.set_device('cpu') self.seed = 2021 + self._init_config() + + def _init_config(self): + self.net = SequentialNet(BufferLayers, 10, 3) + self.model_path = './sequential_net' def _init_seed(self): paddle.seed(self.seed) @@ -58,13 +78,12 @@ def _init_seed(self): def _run(self, to_static): self._init_seed() - net = SequentialNet(BufferLayers, 10, 3) if to_static: - net = paddle.jit.to_static(net) + self.net = paddle.jit.to_static(self.net) x = paddle.rand([16, 10], 'float32') - out = net(x) + out = self.net(x) if to_static: - load_out = self._test_load(net, x) + load_out = self._test_load(self.net, x) self.assertTrue( np.allclose(load_out, out), msg='load_out is {}\st_out is {}'.format(load_out, out)) @@ -80,12 +99,17 @@ def test_train(self): msg='dygraph_res is {}\nstatic_res is {}'.format(dy_out, st_out)) def _test_load(self, net, x): - model_path = './sequential_net' - paddle.jit.save(net, model_path) - load_net = paddle.jit.load(model_path) + paddle.jit.save(net, self.model_path) + load_net = paddle.jit.load(self.model_path) out = load_net(x) return out +class TestNestSequential(TestSequential): + def _init_config(self): + self.net = NestSequentialNet() + self.model_path = './nested_sequential_net' + + if __name__ == '__main__': unittest.main() From 0f5e0ba1ba3042dc4ef53ddb372cd162a42e9d4d Mon Sep 17 00:00:00 2001 From: chentianyu03 Date: Wed, 21 Jul 2021 15:06:19 +0800 Subject: [PATCH 154/156] =?UTF-8?q?=20=E3=80=90cherry-pick=E3=80=91add=20m?= =?UTF-8?q?ore=20info=20to=20tensor.grad=20warning=20message=20(#34264)=20?= =?UTF-8?q?#34288?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add more information to tensor.grad warning message. --- python/paddle/fluid/dygraph/varbase_patch_methods.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index 17cd499bfee5f..2fda67e891abf 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -371,7 +371,10 @@ def grad(self): # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=False, [500.]) """ - msg = "tensor.grad will return the tensor value of the gradient." + msg = 'tensor.grad will return the tensor value of the gradient.' \ + ' This is an incompatible upgrade for tensor.grad API. ' \ + ' It\'s return type changes from numpy.ndarray in version 2.0 to paddle.Tensor in version 2.1.0. ' \ + ' If you want to get the numpy value of the gradient, you can use :code:`x.grad.numpy()`' warning_msg = "\033[93m\nWarning:\n%s \033[0m" % (msg) # ensure ANSI escape sequences print correctly in cmd and powershell if sys.platform.lower() == 'win32': From 2041a0dc7d99b792c30a5ff850e9525005b44ae7 Mon Sep 17 00:00:00 2001 From: dengkaipeng Date: Fri, 30 Jul 2021 04:36:05 +0000 Subject: [PATCH 155/156] fix dataloader exit terminate error. test=develop --- .../fluid/dataloader/dataloader_iter.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index 1f928bfc8a689..b5cfb66577e6b 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -46,6 +46,36 @@ __all__ = ['get_worker_info'] +# NOTE: fix `terminate called without an active exception` +# if for loop break and program exit immediately(with no model +# layers processing) after iterate **the first few data** in +# distributed lauch mode, distributed launch will call +# terminate() to kill main process on each devices, but thread +# is still iterating to fullfill blocking queue caches, which +# may cause thread error `terminate called without an active +# exception` for terminate is a strong singal and `__del__` +# of DataLoader may not be called, so we add a global link to +# the last DataLoader instance to call `__del__` to clean up +# resources +# NOTE: cannot simply as `__del__` to CleanupFuncRegistrar, +# for this will remain a link to each DataLoader instance in +# global, and will precludes GC to auto collect DataLoader +# instance and will cause memory leak +_loader = None + +def _clear_loader(): + global _loader + try: + if _loader: + _loader.__del__() + except: + pass + _loader = None + + +CleanupFuncRegistrar.register(_clear_loader) + + class _DataLoaderIterBase(object): """ Iterator implement of DataLoader, will load and feed mini-batch @@ -90,6 +120,20 @@ def __init__(self, loader): self._thread = None self._thread_done_event = threading.Event() + # record the last DataLoader instance for resource cleaning + global _loader + _loader = self + + @property + def _index_sampler(self): + if self._auto_collate_batch: + return self._batch_sampler + else: + if self._dataset_kind == _DatasetKind.MAP: + return list(range(len(self._dataset))) + else: + return _InfiniteIterableSampler(self._dataset, 1) + def __iter__(self): return self From 7674f5158dbc3ee7af2eac1cdd2ca102cf005c02 Mon Sep 17 00:00:00 2001 From: dengkaipeng Date: Fri, 30 Jul 2021 05:00:45 +0000 Subject: [PATCH 156/156] fix format. test=develop --- python/paddle/fluid/dataloader/dataloader_iter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index b5cfb66577e6b..34f50e9376bdf 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -45,7 +45,6 @@ __all__ = ['get_worker_info'] - # NOTE: fix `terminate called without an active exception` # if for loop break and program exit immediately(with no model # layers processing) after iterate **the first few data** in @@ -63,6 +62,7 @@ # instance and will cause memory leak _loader = None + def _clear_loader(): global _loader try: