Skip to content

2355 parameter groups #2356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wyli merged 1 commit into from
Jun 11, 2021
Merged

2355 parameter groups #2356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions monai/optimizers/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ def generate_param_groups(
layer_matches: a list of callable functions to select or filter out network layer groups,
for "select" type, the input will be the `network`, for "filter" type,
the input will be every item of `network.named_parameters()`.
for "select", the parameters will be
`select_func(network).parameters()`.
for "filter", the parameters will be
`map(lambda x: x[1], filter(filter_func, network.named_parameters()))`
match_types: a list of tags to identify the matching type corresponding to the `layer_matches` functions,
can be "select" or "filter".
lr_values: a list of LR values corresponding to the `layer_matches` functions.
Expand All @@ -48,7 +52,7 @@ def generate_param_groups(
print(net.named_parameters()) # print out all the named parameters to filter out expected items
params = generate_param_groups(
network=net,
layer_matches=[lambda x: x.model[-1], lambda x: "conv.weight" in x],
layer_matches=[lambda x: x.model[0], lambda x: "2.0.conv" in x[0]],
Comment thread
wyli marked this conversation as resolved.
match_types=["select", "filter"],
lr_values=[1e-2, 1e-3],
)
Expand All @@ -71,7 +75,8 @@ def _select():

def _get_filter(f):
def _filter():
return filter(f, network.named_parameters())
# should eventually generate a list of network parameters
return map(lambda x: x[1], filter(f, network.named_parameters()))

return _filter

Expand Down
24 changes: 20 additions & 4 deletions tests/test_generate_param_groups.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"lr_values": [1],
},
(1, 100),
[5, 21],
]

TEST_CASE_2 = [
Expand All @@ -34,6 +35,7 @@
"lr_values": [1, 2, 3],
},
(1, 2, 3, 100),
[5, 16, 5, 0],
]

TEST_CASE_3 = [
Expand All @@ -43,15 +45,17 @@
"lr_values": [1],
},
(1, 100),
[2, 24],
]

TEST_CASE_4 = [
{
"layer_matches": [lambda x: x.model[-1], lambda x: "conv.weight" in x],
"layer_matches": [lambda x: x.model[0], lambda x: "2.0.conv" in x[0]],
"match_types": ["select", "filter"],
"lr_values": [1, 2],
},
(1, 2, 100),
[5, 4, 17],
]

TEST_CASE_5 = [
Expand All @@ -62,12 +66,24 @@
"include_others": False,
},
(1),
[5],
]

TEST_CASE_6 = [
{
"layer_matches": [lambda x: "weight" in x[0]],
"match_types": ["filter"],
"lr_values": [1],
"include_others": True,
},
(1),
[16, 10],
]


class TestGenerateParamGroups(unittest.TestCase):
@parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5])
def test_lr_values(self, input_param, expected_values):
@parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6])
def test_lr_values(self, input_param, expected_values, expected_groups):
device = "cuda" if torch.cuda.is_available() else "cpu"
net = Unet(
dimensions=3,
Expand All @@ -85,7 +101,7 @@ def test_lr_values(self, input_param, expected_values):
torch.testing.assert_allclose(param_group["lr"], value)

n = [len(p["params"]) for p in params]
assert sum(n) == 26 or all(n), "should have either full model or non-empty subsets."
self.assertListEqual(n, expected_groups)

def test_wrong(self):
"""overlapped"""
Expand Down