-
Notifications
You must be signed in to change notification settings - Fork 5.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Enhance ops to support LoD as input for dygraph detection models. #25316
Changes from 2 commits
792d51f
f66a5a3
fe149b8
c950b9d
ac03701
e690dc0
dbb9b99
32cb889
f098394
ce62886
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -65,6 +65,8 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel<T> { | |
|
||
auto multi_layer_scores = | ||
context.MultiInput<paddle::framework::LoDTensor>("MultiLevelScores"); | ||
auto multi_rois_num = context.MultiInput<Tensor>("MultiLevelNums"); | ||
int num_size = multi_rois_num.size(); | ||
|
||
auto* fpn_rois = context.Output<paddle::framework::LoDTensor>("FpnRois"); | ||
|
||
|
@@ -88,23 +90,43 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel<T> { | |
const int num_fpn_level = multi_layer_rois.size(); | ||
std::vector<int> integral_of_all_rois(num_fpn_level + 1, 0); | ||
for (int i = 0; i < num_fpn_level; ++i) { | ||
auto cur_rois_lod = multi_layer_rois[i]->lod().back(); | ||
integral_of_all_rois[i + 1] = | ||
integral_of_all_rois[i] + cur_rois_lod[cur_rois_lod.size() - 1]; | ||
int all_rois = 0; | ||
if (num_size == 0) { | ||
auto cur_rois_lod = multi_layer_rois[i]->lod().back(); | ||
all_rois = cur_rois_lod[cur_rois_lod.size() - 1]; | ||
} else { | ||
const int* cur_rois_num = multi_rois_num[i]->data<int>(); | ||
for (int k = 0; k < multi_rois_num[i]->numel(); k++) { | ||
all_rois += cur_rois_num[k]; | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you also can use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done, thx |
||
} | ||
integral_of_all_rois[i + 1] = integral_of_all_rois[i] + all_rois; | ||
} | ||
|
||
const int batch_size = (num_size == 0) | ||
? multi_layer_rois[0]->lod().back().size() - 1 | ||
: multi_rois_num[0]->numel(); | ||
// concatenate all fpn rois scores into a list | ||
// create a vector to store all scores | ||
std::vector<ScoreWithID<T>> scores_of_all_rois( | ||
integral_of_all_rois[num_fpn_level], ScoreWithID<T>()); | ||
for (int i = 0; i < num_fpn_level; ++i) { | ||
const T* cur_level_scores = multi_layer_scores[i]->data<T>(); | ||
int cur_level_num = integral_of_all_rois[i + 1] - integral_of_all_rois[i]; | ||
auto cur_scores_lod = multi_layer_scores[i]->lod().back(); | ||
int cur_batch_id = 0; | ||
int pre_num = 0; | ||
for (int j = 0; j < cur_level_num; ++j) { | ||
if (static_cast<size_t>(j) >= cur_scores_lod[cur_batch_id + 1]) { | ||
cur_batch_id++; | ||
if (num_size == 0) { | ||
auto cur_scores_lod = multi_layer_scores[i]->lod().back(); | ||
if (static_cast<size_t>(j) >= cur_scores_lod[cur_batch_id + 1]) { | ||
cur_batch_id++; | ||
} | ||
} else { | ||
const int* rois_num_data = multi_rois_num[i]->data<int>(); | ||
if (j >= pre_num + rois_num_data[cur_batch_id]) { | ||
pre_num += rois_num_data[cur_batch_id]; | ||
cur_batch_id++; | ||
} | ||
} | ||
int cur_index = j + integral_of_all_rois[i]; | ||
scores_of_all_rois[cur_index].score = cur_level_scores[j]; | ||
|
@@ -134,6 +156,9 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel<T> { | |
T* fpn_rois_data = fpn_rois->data<T>(); | ||
std::vector<size_t> lod0(1, 0); | ||
int cur_batch_id = 0; | ||
std::vector<int64_t> num_per_batch; | ||
int pre_idx = 0; | ||
int cur_num = 0; | ||
for (int i = 0; i < post_nms_topN; ++i) { | ||
int cur_fpn_level = scores_of_all_rois[i].level; | ||
int cur_level_index = scores_of_all_rois[i].index; | ||
|
@@ -144,6 +169,18 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel<T> { | |
if (scores_of_all_rois[i].batch_id != cur_batch_id) { | ||
cur_batch_id = scores_of_all_rois[i].batch_id; | ||
lod0.emplace_back(i); | ||
cur_num = i - pre_idx; | ||
pre_idx = i; | ||
num_per_batch.emplace_back(cur_num); | ||
} | ||
} | ||
num_per_batch.emplace_back(post_nms_topN - pre_idx); | ||
if (context.HasOutput("RoisNum")) { | ||
auto* rois_num = context.Output<Tensor>("RoisNum"); | ||
int* rois_num_data = | ||
rois_num->mutable_data<int>({batch_size}, context.GetPlace()); | ||
for (int i = 0; i < batch_size; i++) { | ||
rois_num_data[i] = num_per_batch[i]; | ||
} | ||
} | ||
lod0.emplace_back(post_nms_topN); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -48,6 +48,15 @@ class DistributeFpnProposalsOp : public framework::OperatorWithKernel { | |
} | ||
ctx->SetOutputsDim("MultiFpnRois", outs_dims); | ||
ctx->SetOutputDim("RestoreIndex", {-1, 1}); | ||
|
||
if (ctx->HasOutputs("MultiRoisNum")) { | ||
std::vector<framework::DDim> outs_num_dims; | ||
for (size_t i = 0; i < num_out_rois; ++i) { | ||
framework::DDim out_num_dim = {-1}; | ||
outs_num_dims.push_back(out_num_dim); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 55行可以去掉, 56行: push_back({-1}); There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
} | ||
ctx->SetOutputsDim("MultiRoisNum", outs_num_dims); | ||
} | ||
if (!ctx->IsRuntime()) { | ||
for (size_t i = 0; i < num_out_rois; ++i) { | ||
ctx->SetLoDLevel("MultiFpnRois", ctx->GetLoDLevel("FpnRois"), i); | ||
|
@@ -66,12 +75,21 @@ class DistributeFpnProposalsOp : public framework::OperatorWithKernel { | |
class DistributeFpnProposalsOpMaker : public framework::OpProtoAndCheckerMaker { | ||
public: | ||
void Make() override { | ||
AddInput("FpnRois", "(LoDTensor) The rois at all levels in shape (-1, 4)"); | ||
AddInput("FpnRois", "(LoDTensor) The RoIs at all levels in shape (-1, 4)"); | ||
AddInput("RoisNum", | ||
"(Tensor) The number of RoIs in shape (B)," | ||
"B is the number of images") | ||
.AsDispensable(); | ||
AddOutput("MultiFpnRois", "(LoDTensor) Output with distribute operator") | ||
.AsDuplicable(); | ||
AddOutput("RestoreIndex", | ||
"(Tensor) An array of positive number which is " | ||
"used to restore the order of FpnRois"); | ||
AddOutput("MultiRoisNum", | ||
"(Tensor) Multiple number of RoIs from each level in shape (B)," | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这是一个List of Tensor吧,而不是一个Tensor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
"B is the number of images.") | ||
.AsDuplicable() | ||
.AsDispensable(); | ||
AddAttr<int>("min_level", | ||
"The lowest level of FPN layer where the" | ||
" proposals come from"); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -28,6 +28,21 @@ namespace operators { | |
|
||
const int kBoxDim = 4; | ||
|
||
inline std::vector<size_t> get_lod_from_rois_num(const Tensor* rois_num) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Note code style for function-name https://google.github.io/styleguide/cppguide.html#Function_Names There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done, thx |
||
std::vector<size_t> rois_lod; | ||
auto* rois_num_data = rois_num->data<int>(); | ||
Tensor cpu_tensor; | ||
if (platform::is_gpu_place(rois_num->place())) { | ||
TensorCopySync(*rois_num, platform::CPUPlace(), &cpu_tensor); | ||
rois_num_data = cpu_tensor.data<int>(); | ||
} | ||
rois_lod.push_back(static_cast<size_t>(0)); | ||
for (int i = 0; i < rois_num->numel(); ++i) { | ||
rois_lod.push_back(rois_lod.back() + static_cast<size_t>(rois_num_data[i])); | ||
} | ||
return rois_lod; | ||
} | ||
|
||
template <typename T> | ||
static inline T BBoxArea(const T* box, bool normalized) { | ||
if (box[2] < box[0] || box[3] < box[1]) { | ||
|
@@ -65,13 +80,22 @@ class DistributeFpnProposalsOpKernel : public framework::OpKernel<T> { | |
const int num_level = max_level - min_level + 1; | ||
|
||
// check that the fpn_rois is not empty | ||
PADDLE_ENFORCE_EQ( | ||
fpn_rois->lod().size(), 1UL, | ||
platform::errors::InvalidArgument("DistributeFpnProposalsOp needs LoD " | ||
"with one level.")); | ||
if (!context.HasInput("RoisNum")) { | ||
PADDLE_ENFORCE_EQ(fpn_rois->lod().size(), 1UL, | ||
platform::errors::InvalidArgument( | ||
"DistributeFpnProposalsOp needs LoD " | ||
"with one level.")); | ||
} | ||
|
||
auto fpn_rois_lod = fpn_rois->lod().back(); | ||
int fpn_rois_num = fpn_rois_lod[fpn_rois_lod.size() - 1]; | ||
std::vector<size_t> fpn_rois_lod; | ||
int fpn_rois_num; | ||
if (context.HasInput("RoisNum")) { | ||
auto* rois_num = context.Input<Tensor>("RoisNum"); | ||
fpn_rois_lod = get_lod_from_rois_num(rois_num); | ||
} else { | ||
fpn_rois_lod = fpn_rois->lod().back(); | ||
} | ||
fpn_rois_num = fpn_rois_lod[fpn_rois_lod.size() - 1]; | ||
std::vector<int> target_level; | ||
// std::vector<int> target_level(fpn_rois_num, -1); | ||
// record the number of rois in each level | ||
|
@@ -136,6 +160,18 @@ class DistributeFpnProposalsOpKernel : public framework::OpKernel<T> { | |
for (int i = 0; i < fpn_rois_num; ++i) { | ||
restore_index_data[restore_index_inter[i]] = i; | ||
} | ||
auto multi_rois_num = context.MultiOutput<Tensor>("MultiRoisNum"); | ||
if (multi_rois_num.size() > 0) { | ||
int batch_size = fpn_rois_lod.size() - 1; | ||
for (int i = 0; i < num_level; ++i) { | ||
int* rois_num_data = multi_rois_num[i]->mutable_data<int>( | ||
{batch_size}, context.GetPlace()); | ||
for (int j = 0; j < batch_size; ++j) { | ||
rois_num_data[j] = static_cast<int>(multi_fpn_rois_lod0[i][j + 1] - | ||
multi_fpn_rois_lod0[i][j]); | ||
} | ||
} | ||
} | ||
// merge lod information into LoDTensor | ||
for (int i = 0; i < num_level; ++i) { | ||
framework::LoD lod; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -347,7 +347,7 @@ class GenerateProposalsKernel : public framework::OpKernel<T> { | |
lod0.push_back(0); | ||
anchors.Resize({anchors.numel() / 4, 4}); | ||
variances.Resize({variances.numel() / 4, 4}); | ||
std::vector<int64_t> tmp_lod; | ||
std::vector<int> tmp_num; | ||
|
||
int64_t num_proposals = 0; | ||
for (int64_t i = 0; i < num; ++i) { | ||
|
@@ -369,16 +369,16 @@ class GenerateProposalsKernel : public framework::OpKernel<T> { | |
AppendProposals(rpn_roi_probs, num_proposals, scores); | ||
num_proposals += proposals.dims()[0]; | ||
lod0.push_back(num_proposals); | ||
tmp_lod.push_back(num_proposals); | ||
tmp_num.push_back(proposals.dims()[0]); | ||
} | ||
if (context.HasOutput("RpnRoisLod")) { | ||
auto *rpn_rois_lod = context.Output<Tensor>("RpnRoisLod"); | ||
rpn_rois_lod->mutable_data<int64_t>({num}, context.GetPlace()); | ||
int64_t *lod_data = rpn_rois_lod->data<int64_t>(); | ||
if (context.HasOutput("RpnRoisNum")) { | ||
auto *rpn_rois_num = context.Output<Tensor>("RpnRoisNum"); | ||
rpn_rois_num->mutable_data<int>({num}, context.GetPlace()); | ||
int *num_data = rpn_rois_num->data<int>(); | ||
for (int i = 0; i < num; i++) { | ||
lod_data[i] = tmp_lod[i]; | ||
num_data[i] = tmp_num[i]; | ||
} | ||
rpn_rois_lod->Resize({num}); | ||
rpn_rois_num->Resize({num}); | ||
} | ||
rpn_rois->set_lod(lod); | ||
rpn_roi_probs->set_lod(lod); | ||
|
@@ -481,7 +481,8 @@ class GenerateProposalsOpMaker : public framework::OpProtoAndCheckerMaker { | |
"(LoDTensor), Output proposals with shape (rois_num, 4)."); | ||
AddOutput("RpnRoiProbs", | ||
"(LoDTensor) Scores of proposals with shape (rois_num, 1)."); | ||
AddOutput("RpnRoisLod", "(Tensor), rpn rois's lod info").AsDispensable(); | ||
AddOutput("RpnRoisNum", "(Tensor), The number of Rpn RoIs in each image") | ||
.AsDispensable(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 为什么将输出变量RpnRoisLod删除,并新增RpnRoisNum呢? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 关于删除RoisLod相关输入输出问题统一回复:为多个相关api, op的接口保持统一,而且实际输入的值的含义并不是lod而是num。修改后的op主要是为动态图(不支持lod的地方用的),目前release的模型都是静态图版本。同时这些op都是检测模型专有op,其他场景也基本不会用到。 |
||
AddAttr<int>("pre_nms_topN", | ||
"Number of top scoring RPN proposals to keep before " | ||
"applying NMS."); | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
comments不通顺。
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done