Skip to content

Commit 327d178

Browse files
pobonomopobonomo
committed
Add softmax and sigmoid to Keras and Pytorch network
Missing tests and documentation
1 parent 172c02a commit 327d178

File tree

6 files changed

+483
-27
lines changed

6 files changed

+483
-27
lines changed

‎Generate_keras_test_network.ipynb‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "c8d57e80-9075-4d63-bc36-f9aaad08ea2f",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import tensorflow as tf"
11+
]
12+
},
13+
{
14+
"cell_type": "code",
15+
"execution_count": null,
16+
"id": "5d98d000-661e-4495-bef0-49c5eb180aff",
17+
"metadata": {},
18+
"outputs": [],
19+
"source": [
20+
"nn = tf.keras.models.Sequential(\n",
21+
" [\n",
22+
" tf.keras.layers.InputLayer((8,)),\n",
23+
" tf.keras.layers.Dense(30, activation='relu'),\n",
24+
" tf.keras.layers.Dense(1),\n",
25+
" ]\n",
26+
")"
27+
]
28+
},
29+
{
30+
"cell_type": "code",
31+
"execution_count": null,
32+
"id": "ba3cf3ee-bd25-4180-95c0-2ff42d858a34",
33+
"metadata": {},
34+
"outputs": [],
35+
"source": [
36+
"nn.compile(loss='mean_squared_error', optimizer='adam')"
37+
]
38+
},
39+
{
40+
"cell_type": "code",
41+
"execution_count": null,
42+
"id": "247bd200-8026-4f08-8739-9aabb3c37e99",
43+
"metadata": {},
44+
"outputs": [],
45+
"source": [
46+
"(X_train, y_train), (X_test, y_test) = tf.keras.datasets.california_housing.load_data(\n",
47+
" version=\"large\"\n",
48+
")\n"
49+
]
50+
},
51+
{
52+
"cell_type": "code",
53+
"execution_count": null,
54+
"id": "a29325dd-1ab1-4cce-81c0-2528e892adb6",
55+
"metadata": {},
56+
"outputs": [],
57+
"source": [
58+
"normalize = tf.keras.layers.Normalization(axis=-1)"
59+
]
60+
},
61+
{
62+
"cell_type": "code",
63+
"execution_count": null,
64+
"id": "cbecbd91-e100-4568-9424-efd9e3b6d5fc",
65+
"metadata": {},
66+
"outputs": [],
67+
"source": [
68+
"normalize.adapt(X_train)\n",
69+
"X_train = normalize(X_train)\n",
70+
"X_test = normalize(X_test)"
71+
]
72+
},
73+
{
74+
"cell_type": "code",
75+
"execution_count": null,
76+
"id": "5656d2da-ee2d-4a8f-aef3-65876c20193b",
77+
"metadata": {},
78+
"outputs": [],
79+
"source": [
80+
"nn.fit(X_train, y_train, epochs=100, validation_data=(X_test, y_test))"
81+
]
82+
},
83+
{
84+
"cell_type": "code",
85+
"execution_count": null,
86+
"id": "128b1ba9-55e9-4d78-9b31-d2a0da9bb165",
87+
"metadata": {},
88+
"outputs": [],
89+
"source": []
90+
}
91+
],
92+
"metadata": {
93+
"kernelspec": {
94+
"display_name": "Python 3 (ipykernel)",
95+
"language": "python",
96+
"name": "python3"
97+
},
98+
"language_info": {
99+
"codemirror_mode": {
100+
"name": "ipython",
101+
"version": 3
102+
},
103+
"file_extension": ".py",
104+
"mimetype": "text/x-python",
105+
"name": "python",
106+
"nbconvert_exporter": "python",
107+
"pygments_lexer": "ipython3",
108+
"version": "3.11.10"
109+
},
110+
"license": {
111+
"full_text": "# Copyright © 2023 Gurobi Optimization, LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# =============================================================================="
112+
}
113+
},
114+
"nbformat": 4,
115+
"nbformat_minor": 5
116+
}

‎notebooks/adversarial/adversarial_keras.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,9 @@
8989
"nn = tf.keras.models.Sequential(\n",
9090
" [\n",
9191
" tf.keras.layers.InputLayer((28 * 28,)),\n",
92-
" tf.keras.layers.Dense(50, activation=\"relu\"),\n",
93-
" tf.keras.layers.Dense(50, activation=\"relu\"),\n",
94-
" tf.keras.layers.Dense(10),\n",
92+
" tf.keras.layers.Dense(20, activation=\"sigmoid\"),\n",
93+
" tf.keras.layers.Dense(20, activation=\"sigmoid\"),\n",
94+
" tf.keras.layers.Dense(10, activation=\"softmax\"),\n",
9595
" ]\n",
9696
")"
9797
]
@@ -118,7 +118,7 @@
118118
"nn.fit(\n",
119119
" x_train,\n",
120120
" y_train,\n",
121-
" epochs=6,\n",
121+
" epochs=4,\n",
122122
" validation_data=(x_test, y_test),\n",
123123
")"
124124
]
@@ -257,7 +257,7 @@
257257
"name": "python",
258258
"nbconvert_exporter": "python",
259259
"pygments_lexer": "ipython3",
260-
"version": "3.11.8"
260+
"version": "3.11.10"
261261
},
262262
"license": {
263263
"full_text": "# Copyright © 2023 Gurobi Optimization, LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# =============================================================================="

‎notebooks/adversarial/adversarial_pytorch.ipynb‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,17 @@
4242
"import torchvision\n",
4343
"from skorch import NeuralNetClassifier\n",
4444
"\n",
45-
"import gurobipy as gp\n",
46-
"\n",
45+
"import gurobipy as gp"
46+
]
47+
},
48+
{
49+
"cell_type": "code",
50+
"execution_count": null,
51+
"metadata": {},
52+
"outputs": [],
53+
"source": [
54+
"%load_ext autoreload\n",
55+
"%autoreload 2\n",
4756
"from gurobi_ml import add_predictor_constr"
4857
]
4958
},
@@ -100,9 +109,9 @@
100109
"nn_model = torch.nn.Sequential(\n",
101110
" torch.nn.Linear(28 * 28, 50),\n",
102111
" torch.nn.ReLU(),\n",
103-
" torch.nn.Linear(50, 50),\n",
104-
" torch.nn.ReLU(),\n",
105-
" torch.nn.Linear(50, 10),\n",
112+
" torch.nn.Linear(50, 20),\n",
113+
" torch.nn.Sigmoid(),\n",
114+
" torch.nn.Linear(20, 10),\n",
106115
" torch.nn.Softmax(1),\n",
107116
")"
108117
]
@@ -139,7 +148,9 @@
139148
"metadata": {},
140149
"outputs": [],
141150
"source": [
142-
"nn_regression = torch.nn.Sequential(*nn_model[:-1])"
151+
"imageno = 10000\n",
152+
"image = mnist_train.data[imageno, :]\n",
153+
"plt.imshow(image, cmap=\"gray\")"
143154
]
144155
},
145156
{
@@ -148,9 +159,7 @@
148159
"metadata": {},
149160
"outputs": [],
150161
"source": [
151-
"imageno = 10000\n",
152-
"image = mnist_train.data[imageno, :]\n",
153-
"plt.imshow(image, cmap=\"gray\")"
162+
"ex_prob = nn_model.forward(x_train[imageno:imageno+1, :])[0]"
154163
]
155164
},
156165
{
@@ -159,7 +168,6 @@
159168
"metadata": {},
160169
"outputs": [],
161170
"source": [
162-
"ex_prob = nn_regression.forward(x_train[imageno, :])\n",
163171
"sorted_labels = torch.argsort(ex_prob)\n",
164172
"right_label = sorted_labels[-1]\n",
165173
"wrong_label = sorted_labels[-2]"
@@ -188,7 +196,7 @@
188196
"m.addConstr(abs_diff >= -x + image)\n",
189197
"m.addConstr(abs_diff.sum() <= delta)\n",
190198
"\n",
191-
"pred_constr = add_predictor_constr(m, nn_regression, x, y)\n",
199+
"pred_constr = add_predictor_constr(m, nn_model, x, y)\n",
192200
"\n",
193201
"pred_constr.print_stats()"
194202
]
@@ -199,11 +207,19 @@
199207
"metadata": {},
200208
"outputs": [],
201209
"source": [
202-
"m.Params.BestBdStop = 0.0\n",
203-
"m.Params.BestObjStop = 0.0\n",
210+
"m.Params.Obbt = 3\n",
204211
"m.optimize()"
205212
]
206213
},
214+
{
215+
"cell_type": "code",
216+
"execution_count": null,
217+
"metadata": {},
218+
"outputs": [],
219+
"source": [
220+
"pred_constr.get_error()"
221+
]
222+
},
207223
{
208224
"cell_type": "code",
209225
"execution_count": null,

‎src/gurobi_ml/keras/keras.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ def __init__(self, gp_model, predictor, input_vars, output_vars=None, **kwargs):
7777
if isinstance(step, keras.layers.Dense):
7878
config = step.get_config()
7979
activation = config["activation"]
80-
if activation not in ("relu", "linear"):
80+
if activation not in ("relu", "softmax", "sigmoid", "linear"):
8181
raise NoModel(predictor, f"Unsupported activation {activation}")
8282
elif isinstance(step, keras.layers.ReLU):
8383
if step.negative_slope != 0.0:
@@ -120,6 +120,8 @@ def _mip_model(self, **kwargs):
120120
activation = config["activation"]
121121
if activation == "linear":
122122
activation = "identity"
123+
if activation == "sigmoid":
124+
activation = "logistic"
123125
weights, bias = step.get_weights()
124126
layer = self._add_dense_layer(
125127
_input,

‎src/gurobi_ml/torch/sequential.py‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,15 @@ class SequentialConstr(BaseNNConstr):
7676
|ClassShort|.
7777
"""
7878

79+
activations = {
80+
nn.ReLU: "relu",
81+
nn.Softmax: "softmax",
82+
nn.Sigmoid: "logistic",
83+
}
84+
7985
def __init__(self, gp_model, predictor, input_vars, output_vars=None, **kwargs):
8086
for step in predictor:
81-
if isinstance(step, nn.ReLU):
87+
if isinstance(step, tuple(self.activations.keys())):
8288
pass
8389
elif isinstance(step, nn.Linear):
8490
pass
@@ -95,12 +101,7 @@ def _mip_model(self, **kwargs):
95101
for i, step in enumerate(network):
96102
if i == num_layers - 1:
97103
output = self._output
98-
if isinstance(step, nn.ReLU):
99-
layer = self._add_activation_layer(
100-
_input, self.act_dict["relu"](), output, name=f"relu_{i}", **kwargs
101-
)
102-
_input = layer.output
103-
elif isinstance(step, nn.Linear):
104+
if isinstance(step, nn.Linear):
104105
layer_weight = None
105106
layer_bias = None
106107
for name, param in step.named_parameters():
@@ -122,7 +123,17 @@ def _mip_model(self, **kwargs):
122123
**kwargs,
123124
)
124125
_input = layer.output
125-
if self._output is None:
126+
else:
127+
activation = self.activations[type(step)]
128+
layer = self._add_activation_layer(
129+
_input,
130+
self.act_dict[activation](),
131+
output,
132+
name=f"{activation}_{i}",
133+
**kwargs,
134+
)
135+
_input = layer.output
136+
if self.output is None:
126137
self._output = layer.output
127138

128139
def get_error(self, eps=None):

0 commit comments

Comments
 (0)