Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Cherry-pick]Delete the function of saving layer object. #34039

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 2 additions & 12 deletions python/paddle/fluid/tests/unittests/test_paddle_save_load.py
Original file line number Diff line number Diff line change
Expand Up @@ -869,21 +869,11 @@ def test_save_load_layer(self):
layer2 = LinearNet()
layer1.eval()
layer2.eval()
origin_layer = (layer1, layer2)
origin = (layer1(inps), layer2(inps))
path = "test_save_load_layer_/layer.pdmodel"
paddle.save((layer1, layer2), path)

# static
paddle.enable_static()
with self.assertRaises(ValueError):
paddle.load(path)
# dygraph
paddle.disable_static()

loaded_layer = paddle.load(path)
loaded_result = [l(inps) for l in loaded_layer]
for i in range(len(origin)):
self.assertTrue((origin[i] - loaded_result[i]).abs().max() < 1e-10)
paddle.save(origin_layer, path)


if __name__ == '__main__':
Expand Down
32 changes: 24 additions & 8 deletions python/paddle/framework/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ def _pickle_save(obj, f, protocol):
raise ValueError("Expected 1<'protocol'<5, but received protocol={}".
format(protocol))

def reudce_varbase(self):
def reduce_varbase(self):
data = self.numpy()
name = self.name

Expand All @@ -243,16 +243,32 @@ def reduce_LoDTensor(self):

return (eval, ('data', {'data': data}))

def reduce_Layer(self):
raise ValueError(
"paddle do not support saving `paddle.nn.Layer` object.")

dispatch_table_layer = dict()

def create_layer_dispatch_table(layer):
dispatch_table_layer[layer.__class__] = reduce_Layer
return layer

_parse_every_object(obj, lambda v: isinstance(v, core.Layer),
create_layer_dispatch_table)

def add_dispatch_table():
# This is not a good method, because the pickle module has been modified.
pickle.dispatch_table[core.VarBase] = reudce_varbase
pickle.dispatch_table[ParamBase] = reudce_varbase
pickle.dispatch_table[core.VarBase] = reduce_varbase
pickle.dispatch_table[ParamBase] = reduce_varbase
pickle.dispatch_table[core.LoDTensor] = reduce_LoDTensor
pickle.dispatch_table.update(dispatch_table_layer)

def pop_dispatch_table():
pickle.dispatch_table.pop(core.VarBase)
pickle.dispatch_table.pop(core.LoDTensor)
pickle.dispatch_table.pop(ParamBase)
for k in dispatch_table_layer:
pickle.dispatch_table.pop(k)

# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if sys.platform == 'darwin' and sys.version_info.major == 3:
Expand All @@ -272,10 +288,10 @@ def pop_dispatch_table():
pickler = pickle.Pickler(f, protocol)
pickler.dispatch_table = copyreg.dispatch_table.copy()

pickler.dispatch_table[core.VarBase] = reudce_varbase
pickler.dispatch_table[core.VarBase] = reduce_varbase
pickler.dispatch_table[core.LoDTensor] = reduce_LoDTensor
pickler.dispatch_table[ParamBase] = reudce_varbase

pickler.dispatch_table[ParamBase] = reduce_varbase
pickler.dispatch_table.update(dispatch_table_layer)
pickler.dump(obj)


Expand Down Expand Up @@ -496,7 +512,7 @@ def save(obj, path, protocol=4, **configs):
Save an object to the specified path.

.. note::
Now supports saving ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor, Program.
Now supports saving ``state_dict`` of Layer/Optimizer, Tensor and nested structure containing Tensor, Program.

.. note::
Different from ``paddle.jit.save``, since the save result of ``paddle.save`` is a single file,
Expand Down Expand Up @@ -690,7 +706,7 @@ def load(path, **configs):
Load an object can be used in paddle from specified path.

.. note::
Now supports loading ``state_dict`` of Layer/Optimizer, Layer, Tensor and nested structure containing Tensor, Program.
Now supports loading ``state_dict`` of Layer/Optimizer, Tensor and nested structure containing Tensor, Program.

.. note::
In order to use the model parameters saved by paddle more efficiently,
Expand Down