Examples

This repository contains runnable example scripts in examples/:

example_act.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""Example the usage of activation functions in qkeras."""
 17
 18import numpy as np
 19
 20from qkeras import (
 21    bernoulli,
 22    binary,
 23    hard_sigmoid,
 24    hard_tanh,
 25    quantized_bits,
 26    quantized_po2,
 27    quantized_relu,
 28    quantized_relu_po2,
 29    quantized_tanh,
 30    set_internal_sigmoid,
 31    smooth_sigmoid,
 32    smooth_tanh,
 33    stochastic_binary,
 34    stochastic_ternary,
 35    ternary,
 36)
 37
 38
 39def main():
 40    # check the mean value of samples from stochastic_rounding for po2
 41
 42    count = 100000
 43    val = 42
 44    a = np.array([val] * count)
 45    b = quantized_po2(use_stochastic_rounding=True)(a)
 46    res = np.sum(b.numpy()) / count
 47    print(res, "should be close to ", val)
 48    b = quantized_relu_po2(use_stochastic_rounding=True)(a)
 49    res = np.sum(b.numpy()) / count
 50    print(res, "should be close to ", val)
 51    a = np.array([-1] * count)
 52    b = quantized_relu_po2(use_stochastic_rounding=True)(a)
 53    res = np.sum(b.numpy()) / count
 54    print(res, "should be all ", 0)
 55
 56    # non-stochastic rounding quantizer.
 57    a = np.array([-3.0, -2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0, 3.0])
 58    a = np.array([0.194336])
 59    print(" a =", a.astype("float16"))
 60    print("qa =", quantized_relu(6, 2)(a).numpy().astype("float16"))
 61    print("ss =", smooth_sigmoid(a).numpy().astype("float16"))
 62    print("hs =", hard_sigmoid(a).numpy().astype("float16"))
 63    print("ht =", hard_tanh(a).numpy().astype("float16"))
 64    print("st =", smooth_tanh(a).numpy().astype("float16"))
 65    c = np.array(np.arange(-1.5, 1.51, 0.3), dtype="float32")
 66    print(" c =", c.astype("float16"))
 67    print("qb_111 =", quantized_bits(1, 1, 1)(c).numpy().astype("float16"))
 68    print("qb_210 =", quantized_bits(2, 1, 0)(c).numpy().astype("float16"))
 69    print("qb_211 =", quantized_bits(2, 1, 1)(c).numpy().astype("float16"))
 70    print("qb_300 =", quantized_bits(3, 0, 0)(c).numpy().astype("float16"))
 71    print("qb_301 =", quantized_bits(3, 0, 1)(c).numpy().astype("float16"))
 72
 73    c_1000 = np.array(np.array([list(c)] * 1000), dtype="float32")
 74    b = np.sum(bernoulli()(c_1000).numpy().astype(np.int32), axis=0) / 1000.0
 75    print("       hs =", hard_sigmoid(c).numpy().astype("float16"))
 76    print("    b_all =", b.astype("float16"))
 77
 78    T = 0.0
 79    t = stochastic_ternary(alpha="auto")(c_1000).numpy()
 80    for i in range(10):
 81        print(f"stochastic_ternary({i}) =", t[i])
 82    print(
 83        "   st_all =",
 84        np.round(
 85            np.sum(t.astype("float32"), axis=0).astype("float16") / 1000.0, 2
 86        ).astype("float16"),
 87    )
 88
 89    print("  ternary =", ternary(threshold=0.5)(c).numpy().astype(np.int32))
 90    print(" c =", c.astype("float16"))
 91    print(" b_10 =", binary(1)(c).numpy().astype("float16"))
 92    print("qr_10 =", quantized_relu(1, 0)(c).numpy().astype("float16"))
 93    print("qr_11 =", quantized_relu(1, 1)(c).numpy().astype("float16"))
 94    print("qr_20 =", quantized_relu(2, 0)(c).numpy().astype("float16"))
 95    print("qr_21 =", quantized_relu(2, 1)(c).numpy().astype("float16"))
 96    print("qr_101 =", quantized_relu(1, 0, 1)(c).numpy().astype("float16"))
 97    print("qr_111 =", quantized_relu(1, 1, 1)(c).numpy().astype("float16"))
 98    print("qr_201 =", quantized_relu(2, 0, 1)(c).numpy().astype("float16"))
 99    print("qr_211 =", quantized_relu(2, 1, 1)(c).numpy().astype("float16"))
100    print("qt_200 =", quantized_tanh(2, 0)(c).numpy().astype("float16"))
101    print("qt_210 =", quantized_tanh(2, 1)(c).numpy().astype("float16"))
102    print("qt_201 =", quantized_tanh(2, 0, 1)(c).numpy().astype("float16"))
103    print("qt_211 =", quantized_tanh(2, 1, 1)(c).numpy().astype("float16"))
104
105    set_internal_sigmoid("smooth")
106    print("with smooth sigmoid")
107    print("qr_101 =", quantized_relu(1, 0, 1)(c).numpy().astype("float16"))
108    print("qr_111 =", quantized_relu(1, 1, 1)(c).numpy().astype("float16"))
109    print("qr_201 =", quantized_relu(2, 0, 1)(c).numpy().astype("float16"))
110    print("qr_211 =", quantized_relu(2, 1, 1)(c).numpy().astype("float16"))
111    print("qt_200 =", quantized_tanh(2, 0)(c).numpy().astype("float16"))
112    print("qt_210 =", quantized_tanh(2, 1)(c).numpy().astype("float16"))
113    print("qt_201 =", quantized_tanh(2, 0, 1)(c).numpy().astype("float16"))
114    print("qt_211 =", quantized_tanh(2, 1, 1)(c).numpy().astype("float16"))
115
116    set_internal_sigmoid("real")
117    print("with real sigmoid")
118    print("qr_101 =", quantized_relu(1, 0, 1)(c).numpy().astype("float16"))
119    print("qr_111 =", quantized_relu(1, 1, 1)(c).numpy().astype("float16"))
120    print("qr_201 =", quantized_relu(2, 0, 1)(c).numpy().astype("float16"))
121    print("qr_211 =", quantized_relu(2, 1, 1)(c).numpy().astype("float16"))
122    print("qt_200 =", quantized_tanh(2, 0)(c).numpy().astype("float16"))
123    print("qt_210 =", quantized_tanh(2, 1)(c).numpy().astype("float16"))
124    print("qt_201 =", quantized_tanh(2, 0, 1)(c).numpy().astype("float16"))
125    print("qt_211 =", quantized_tanh(2, 1, 1)(c).numpy().astype("float16"))
126
127    set_internal_sigmoid("hard")
128    print(" c =", c.astype("float16"))
129    print("q2_31 =", quantized_po2(3, 1)(c).numpy().astype("float16"))
130    print("q2_32 =", quantized_po2(3, 2)(c).numpy().astype("float16"))
131    print("qr2_21 =", quantized_relu_po2(2, 1)(c).numpy().astype("float16"))
132    print("qr2_22 =", quantized_relu_po2(2, 2)(c).numpy().astype("float16"))
133    print("qr2_44 =", quantized_relu_po2(4, 1)(c).numpy().astype("float16"))
134
135    print("q2_32_2 =", quantized_relu_po2(32, 2)(c).numpy().astype("float16"))
136
137    b = stochastic_binary()(c_1000).numpy().astype(np.int32)
138    for i in range(5):
139        print(f"sbinary({i}) =", b[i])
140    print("sbinary =", np.round(np.sum(b, axis=0) / 1000.0, 2).astype("float16"))
141    print(" binary =", binary()(c).numpy().astype(np.int32))
142    print(" c      =", c.astype("float16"))
143
144    for i in range(10):
145        print(
146            f" s_bin({i}) =",
147            binary(use_stochastic_rounding=1)(c).numpy().astype(np.int32),
148        )
149    for i in range(10):
150        print(
151            f" s_po2({i}) =",
152            quantized_po2(use_stochastic_rounding=1)(c).numpy().astype(np.int32),
153        )
154    for i in range(10):
155        print(
156            f" s_relu_po2({i}) =",
157            quantized_relu_po2(use_stochastic_rounding=1)(c).numpy().astype(np.int32),
158        )
159
160
161if __name__ == "__main__":
162    main()

example_cifar10_po2.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""Tests qcore model with po2."""
 17
 18
 19import os
 20
 21import keras.ops.numpy as knp
 22from keras.datasets import cifar10
 23from keras.layers import *
 24from keras.models import Model
 25from keras.optimizers import *
 26from keras.utils import to_categorical
 27
 28from qkeras import *
 29
 30NB_EPOCH = 50
 31BATCH_SIZE = 64
 32VERBOSE = 1
 33NB_CLASSES = 10
 34OPTIMIZER = Adam(learning_rate=0.0001)
 35VALIDATION_SPLIT = 0.1
 36
 37(x_train, y_train), (x_test, y_test) = cifar10.load_data()
 38
 39x_train = x_train.astype(float)
 40x_test = x_test.astype(float)
 41
 42x_train /= 255.0
 43x_test /= 255.0
 44
 45print(x_train.shape[0], "train samples")
 46print(x_test.shape[0], "test samples")
 47
 48print(y_train[0:10])
 49
 50y_train = to_categorical(y_train, NB_CLASSES)
 51y_test = to_categorical(y_test, NB_CLASSES)
 52
 53x = x_in = Input(x_train.shape[1:], name="input")
 54x = QActivation("quantized_relu_po2(4,4)", name="acti")(x)
 55x = QConv2D(
 56    128,
 57    (3, 3),
 58    strides=1,
 59    kernel_quantizer=quantized_po2(4, 1),
 60    bias_quantizer=quantized_po2(4, 4),
 61    bias_range=4,
 62    name="conv2d_0_m",
 63)(x)
 64x = QActivation("ternary()", name="act0_m")(x)
 65x = MaxPooling2D(2, 2, name="mp_0")(x)
 66x = QConv2D(
 67    256,
 68    (3, 3),
 69    strides=1,
 70    kernel_quantizer=quantized_po2(4, 1),
 71    bias_quantizer=quantized_po2(4, 4),
 72    bias_range=4,
 73    name="conv2d_1_m",
 74)(x)
 75x = QActivation("quantized_relu(6,2)", name="act1_m")(x)
 76x = MaxPooling2D(2, 2, name="mp_1")(x)
 77x = QConv2D(
 78    128,
 79    (3, 3),
 80    strides=1,
 81    kernel_quantizer=quantized_bits(4, 0, 1),
 82    bias_quantizer=quantized_bits(4, 0, 1),
 83    name="conv2d_2_m",
 84)(x)
 85x = QActivation("quantized_relu(4,2)", name="act2_m")(x)
 86x = MaxPooling2D(2, 2, name="mp_2")(x)
 87x = Flatten()(x)
 88x = QDense(
 89    NB_CLASSES,
 90    kernel_quantizer=quantized_ulaw(4, 0, 1),
 91    bias_quantizer=quantized_bits(4, 0, 1),
 92    name="dense",
 93)(x)
 94x = Activation("softmax", name="softmax")(x)
 95
 96model = Model(inputs=[x_in], outputs=[x])
 97model.summary()
 98
 99model.compile(
100    loss="categorical_crossentropy", optimizer=OPTIMIZER, metrics=["accuracy"]
101)
102
103if int(os.environ.get("TRAIN", 0)):
104    history = model.fit(
105        x_train,
106        y_train,
107        batch_size=BATCH_SIZE,
108        epochs=NB_EPOCH,
109        initial_epoch=1,
110        verbose=VERBOSE,
111        validation_split=VALIDATION_SPLIT,
112    )
113
114    outputs = []
115    output_names = []
116
117    for layer in model.layers:
118        if layer.__class__.__name__ in [
119            "QActivation",
120            "Activation",
121            "QDense",
122            "QConv2D",
123            "QDepthwiseConv2D",
124        ]:
125            output_names.append(layer.name)
126            outputs.append(layer.output)
127
128    model_debug = Model(inputs=[x_in], outputs=outputs)
129
130    outputs = model_debug.predict(x_train)
131
132    print("{:30} {: 8.4f} {: 8.4f}".format("input", knp.min(x_train), knp.max(x_train)))
133
134    for n, p in zip(output_names, outputs):
135        print(f"{n:30} {knp.min(p): 8.4f} {knp.max(p): 8.4f}", end="")
136        layer = model.get_layer(n)
137        for i, weights in enumerate(layer.get_weights()):
138            weights = layer.get_quantizers()[i](weights)
139            print(
140                f" ({knp.min(weights): 8.4f} {knp.max(weights): 8.4f})", end=""
141            )
142            print("")
143
144    score = model.evaluate(x_test, y_test, verbose=VERBOSE)
145    print("Test score:", score[0])
146    print("Test accuracy:", score[1])
147
148model.summary()
149
150print_qstats(model)

example_ternary.py

  1# Copyright 2020 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16
 17import keras.ops.numpy as knp
 18import matplotlib
 19import numpy as np
 20from absl import app, flags
 21
 22matplotlib.use("TkAgg")
 23import matplotlib.pyplot as plt
 24
 25FLAGS = flags.FLAGS
 26
 27
 28def _stochastic_rounding(x, precision, resolution, delta):
 29    """Stochastic_rounding for ternary.
 30
 31    Args:
 32      x:
 33      precision: A float. The area we want to make this stochastic rounding.
 34         [delta-precision, delta] [delta, delta+precision]
 35      resolution: control the quantization resolution.
 36      delta: the undiscountinued point (positive number)
 37
 38    Return:
 39      A tensor with stochastic rounding numbers.
 40    """
 41    delta_left = delta - precision
 42    delta_right = delta + precision
 43    scale = 1 / resolution
 44    scale_delta_left = delta_left * scale
 45    scale_delta_right = delta_right * scale
 46    scale_2_delta = scale_delta_right - scale_delta_left
 47    scale_x = x * scale
 48    fraction = scale_x - scale_delta_left
 49    # print(precision, scale, x[0], knp.floor(scale_x[0]), scale_x[0], fraction[0])
 50
 51    # we use uniform distribution
 52    random_selector = np.random.uniform(0, 1, size=x.shape) * scale_2_delta
 53
 54    # print(precision, scale, x[0], delta_left[0], delta_right[0])
 55    # print('x', scale_x[0], fraction[0], random_selector[0], scale_2_delta[0])
 56    # rounddown = fraction < random_selector
 57    result = knp.where(
 58        fraction < random_selector, scale_delta_left / scale, scale_delta_right / scale
 59    )
 60    return result
 61
 62
 63def _ternary(x, sto=False):
 64    m = knp.amax(knp.abs(x), keepdims=True)
 65    scale = 2 * m / 3.0
 66    thres = scale / 2.0
 67    ratio = 0.1
 68
 69    if sto:
 70        sign_bit = knp.sign(x)
 71        x = knp.abs(x)
 72        prec = x / scale
 73        x = (
 74            sign_bit
 75            * scale
 76            * _stochastic_rounding(
 77                x / scale,
 78                precision=0.3,
 79                resolution=0.01,  # those two are all normalized.
 80                delta=thres / scale,
 81            )
 82        )
 83        # prec + prec *ratio)
 84        # mm = knp.amax(knp.abs(x), keepdims=True)
 85    return knp.where(knp.abs(x) < thres, knp.zeros_like(x), knp.sign(x))
 86
 87
 88def main(argv):
 89    if len(argv) > 1:
 90        raise app.UsageError("Too many command-line arguments.")
 91
 92    # x = knp.arange(-3.0, 3.0, 0.01)
 93    # x = np.random.uniform(-0.01, 0.01, size=1000)
 94    x = np.random.uniform(-10.0, 10.0, size=1000)
 95    # x = np.random.uniform(-1, 1, size=1000)
 96    x = knp.sort(x)
 97    tr = knp.zeros_like(x)
 98    t = knp.zeros_like(x)
 99    iter_count = 500
100    for _ in range(iter_count):
101        y = _ternary(x)
102        yr = _ternary(x, sto=True)
103        t = t + y
104        tr = tr + yr
105
106    plt.plot(x, t / iter_count)
107    plt.plot(x, tr / iter_count)
108    plt.ylabel("mean (%s samples)" % iter_count)
109    plt.show()
110
111
112if __name__ == "__main__":
113    app.run(main)

example_mnist.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""uses po2."""
 17
 18
 19
 20import keras.ops.numpy as knp
 21from keras.datasets import mnist
 22from keras.layers import *
 23from keras.layers import Activation, Flatten, Input
 24from keras.models import Model
 25from keras.optimizers import Adam
 26from keras.utils import to_categorical
 27
 28from qkeras import *
 29from qkeras.utils import model_save_quantized_weights
 30
 31NB_EPOCH = 10
 32BATCH_SIZE = 64
 33VERBOSE = 1
 34NB_CLASSES = 10
 35OPTIMIZER = Adam(learning_rate=0.0001, decay=0.000025)
 36VALIDATION_SPLIT = 0.1
 37
 38train = 1
 39
 40(x_train, y_train), (x_test, y_test) = mnist.load_data()
 41
 42RESHAPED = 784
 43
 44x_test_orig = x_test
 45
 46x_train = x_train.astype(float)
 47x_test = x_test.astype(float)
 48
 49x_train = x_train[..., np.newaxis]
 50x_test = x_test[..., np.newaxis]
 51
 52x_train /= 256.0
 53x_test /= 256.0
 54
 55print(x_train.shape[0], "train samples")
 56print(x_test.shape[0], "test samples")
 57
 58print(y_train[0:10])
 59
 60y_train = to_categorical(y_train, NB_CLASSES)
 61y_test = to_categorical(y_test, NB_CLASSES)
 62
 63x = x_in = Input(x_train.shape[1:-1] + (1,), name="input")
 64x = QConv2D(
 65    32,
 66    (2, 2),
 67    strides=(2, 2),
 68    kernel_quantizer=quantized_bits(4, 0, 1),
 69    bias_quantizer=quantized_bits(4, 0, 1),
 70    name="conv2d_0_m",
 71)(x)
 72x = QActivation("quantized_relu(4,0)", name="act0_m")(x)
 73x = QConv2D(
 74    64,
 75    (3, 3),
 76    strides=(2, 2),
 77    kernel_quantizer=quantized_bits(4, 0, 1),
 78    bias_quantizer=quantized_bits(4, 0, 1),
 79    name="conv2d_1_m",
 80)(x)
 81x = QActivation("quantized_relu(4,0)", name="act1_m")(x)
 82x = QConv2D(
 83    64,
 84    (2, 2),
 85    strides=(2, 2),
 86    kernel_quantizer=quantized_bits(4, 0, 1),
 87    bias_quantizer=quantized_bits(4, 0, 1),
 88    name="conv2d_2_m",
 89)(x)
 90x = QActivation("quantized_relu(4,0)", name="act2_m")(x)
 91x = Flatten()(x)
 92x = QDense(
 93    NB_CLASSES,
 94    kernel_quantizer=quantized_bits(4, 0, 1),
 95    bias_quantizer=quantized_bits(4, 0, 1),
 96    name="dense",
 97)(x)
 98x_out = x
 99x = Activation("softmax", name="softmax")(x)
100
101model = Model(inputs=[x_in], outputs=[x])
102mo = Model(inputs=[x_in], outputs=[x_out])
103model.summary()
104
105model.compile(
106    loss="categorical_crossentropy", optimizer=OPTIMIZER, metrics=["accuracy"]
107)
108
109if train:
110    history = model.fit(
111        x_train,
112        y_train,
113        batch_size=BATCH_SIZE,
114        epochs=NB_EPOCH,
115        initial_epoch=1,
116        verbose=VERBOSE,
117        validation_split=VALIDATION_SPLIT,
118    )
119
120    outputs = []
121    output_names = []
122
123    for layer in model.layers:
124        if layer.__class__.__name__ in [
125            "QActivation",
126            "Activation",
127            "QDense",
128            "QConv2D",
129            "QDepthwiseConv2D",
130        ]:
131            output_names.append(layer.name)
132            outputs.append(layer.output)
133
134    model_debug = Model(inputs=[x_in], outputs=outputs)
135
136    outputs = model_debug.predict(x_train)
137
138    print("{:30} {: 8.4f} {: 8.4f}".format("input", knp.min(x_train), knp.max(x_train)))
139
140    for n, p in zip(output_names, outputs):
141        print(f"{n:30} {knp.min(p): 8.4f} {knp.max(p): 8.4f}", end="")
142        layer = model.get_layer(n)
143        for i, weights in enumerate(layer.get_weights()):
144            weights = layer.get_quantizers()[i](weights)
145            print(
146                f" ({knp.min(weights): 8.4f} {knp.max(weights): 8.4f})", end=""
147            )
148            print("")
149
150    p_test = mo.predict(x_test)
151    p_test.tofile("p_test.bin")
152
153    score = model.evaluate(x_test, y_test, verbose=VERBOSE)
154    print("Test score:", score[0])
155    print("Test accuracy:", score[1])
156
157    all_weights = []
158    model_save_quantized_weights(model)
159
160    for layer in model.layers:
161        for w, weights in enumerate(layer.get_weights()):
162            print(layer.name, w)
163            all_weights.append(weights.flatten())
164
165    all_weights = np.concatenate(all_weights).astype("float32")
166    print(all_weights.size)
167
168
169for layer in model.layers:
170    for w, weight in enumerate(layer.get_weights()):
171        print(layer.name, w, weight.shape)
172
173print_qstats(model)

example_qoctave.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""QOctave example."""
 17
 18from keras import initializers
 19from keras import ops as Kops
 20from keras.layers import Activation, Input, UpSampling2D
 21from keras.models import Model
 22
 23from qkeras import *  # pylint: disable=wildcard-import
 24
 25
 26def create_model():
 27    """use qocatve in network."""
 28    kernel_initializer = initializers.he_normal(seed=42)
 29
 30    x = x_in = Input(shape=(256, 256, 3))
 31
 32    # Block 1
 33    high, low = QOctaveConv2D(
 34        32,
 35        (3, 3),
 36        alpha=0.5,
 37        strides=(2, 2),
 38        padding="valid",
 39        kernel_initializer=kernel_initializer,
 40        bias_initializer="zeros",
 41        bias_quantizer="quantized_bits(4,1)",
 42        depthwise_quantizer="quantized_bits(4,1)",
 43        depthwise_activation="quantized_bits(6,2,1)",
 44        pointwise_quantizer="quantized_bits(4,1)",
 45        acc_quantizer="quantized_bits(16,7,1)",
 46        activation="quantized_relu(6,2)",
 47        use_separable=True,
 48        name="block1_conv1",
 49    )([x, None])
 50
 51    # Block 2
 52    high, low = QOctaveConv2D(
 53        64,
 54        (3, 3),
 55        alpha=0.4,
 56        strides=(2, 2),
 57        padding="same",
 58        kernel_initializer=kernel_initializer,
 59        bias_initializer="zeros",
 60        bias_quantizer="quantized_bits(4,1)",
 61        depthwise_quantizer="quantized_bits(4,1)",
 62        depthwise_activation="quantized_bits(6,2,1)",
 63        pointwise_quantizer="quantized_bits(4,1)",
 64        acc_quantizer="quantized_bits(16,7,1)",
 65        activation="quantized_relu(6,2)",
 66        use_separable=True,
 67        name="block2_conv1",
 68    )([high, low])
 69
 70    # Block 3
 71    high, low = QOctaveConv2D(
 72        64,
 73        (3, 3),
 74        alpha=0.4,
 75        strides=(2, 2),
 76        padding="same",
 77        kernel_initializer=kernel_initializer,
 78        bias_initializer="zeros",
 79        bias_quantizer="quantized_bits(4,1)",
 80        depthwise_quantizer="quantized_bits(4,1)",
 81        depthwise_activation="quantized_bits(6,2,1)",
 82        pointwise_quantizer="quantized_bits(4,1)",
 83        acc_quantizer="quantized_bits(16,7,1)",
 84        activation="quantized_relu(6,2)",
 85        use_separable=True,
 86        name="block3_conv1",
 87    )([high, low])
 88
 89    high, low = QOctaveConv2D(
 90        32,
 91        (3, 3),
 92        alpha=0.4,
 93        strides=(1, 1),
 94        padding="same",
 95        kernel_initializer=kernel_initializer,
 96        bias_initializer="zeros",
 97        bias_quantizer="quantized_bits(4,1)",
 98        depthwise_quantizer="quantized_bits(4,1)",
 99        depthwise_activation="quantized_bits(6,2,1)",
100        pointwise_quantizer="quantized_bits(4,1)",
101        acc_quantizer="quantized_bits(16,7,1)",
102        activation="quantized_relu(6,2)",
103        use_separable=True,
104        name="block3_conv2",
105    )([high, low])
106
107    high, low = QOctaveConv2D(
108        32,
109        (3, 3),
110        alpha=0.3,
111        strides=(1, 1),
112        padding="same",
113        kernel_initializer=kernel_initializer,
114        bias_initializer="zeros",
115        bias_quantizer="quantized_bits(4,1)",
116        depthwise_quantizer="quantized_bits(4,1)",
117        depthwise_activation="quantized_bits(6,2,1)",
118        pointwise_quantizer="quantized_bits(4,1)",
119        acc_quantizer="quantized_bits(16,7,1)",
120        activation="quantized_relu(6,2)",
121        use_separable=True,
122        name="block3_conv3",
123    )([high, low])
124
125    x, _ = QOctaveConv2D(
126        32,
127        (3, 3),
128        alpha=0.0,
129        strides=(2, 2),
130        padding="same",
131        kernel_initializer=kernel_initializer,
132        bias_initializer="zeros",
133        bias_quantizer="quantized_bits(4,1)",
134        depthwise_quantizer="quantized_bits(4,1)",
135        depthwise_activation="quantized_bits(6,2,1)",
136        pointwise_quantizer="quantized_bits(4,1)",
137        acc_quantizer="quantized_bits(16,7,1)",
138        activation="quantized_relu(6,2)",
139        use_separable=True,
140        name="block3_conv_down",
141    )([high, low])
142
143    # Upsample
144    x = UpSampling2D(size=(2, 2), data_format="channels_last")(x)
145
146    x = QConv2D(
147        2,
148        (2, 2),
149        strides=(1, 1),
150        kernel_initializer=kernel_initializer,
151        bias_initializer="ones",
152        kernel_quantizer=quantized_bits(4, 0, 1),
153        bias_quantizer=quantized_bits(4, 0, 1),
154        padding="same",
155        name="conv_up",
156    )(x)
157
158    x = Activation("softmax", name="softmax")(x)
159    output = x
160
161    model = Model(x_in, output, name="qoctave_network")
162    return model
163
164
165@keras.saving.register_keras_serializable()
166def customLoss(y_true, y_pred):
167    log1 = 1.5 * y_true * Kops.log(y_pred + 1e-9) * keras.ops.power(1 - y_pred, 2)
168    log0 = 0.5 * (1 - y_true) * Kops.log((1 - y_pred) + 1e-9) * keras.ops.power(y_pred, 2)
169    return -Kops.sum(keras.ops.mean(log0 + log1, axis=0))
170
171
172if __name__ == "__main__":
173    model = create_model()
174    model.compile(optimizer="Adam", loss=customLoss, metrics=["acc"])
175    model.summary(line_length=100)
176    print_qstats(model)

example_qdense.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""Tests qdense model."""
 17
 18
 19import argparse
 20
 21from keras.datasets import mnist
 22from keras.layers import Activation, Input
 23from keras.models import Model
 24from keras.optimizers import Adam
 25from keras.utils import to_categorical
 26
 27from qkeras import QActivation, QDense, print_qstats, quantized_bits, ternary
 28
 29OPTIMIZER = Adam()
 30NB_EPOCH = 1
 31BATCH_SIZE = 32
 32VERBOSE = 1
 33NB_CLASSES = 10
 34N_HIDDEN = 100
 35VALIDATION_SPLIT = 0.1
 36RESHAPED = 784
 37
 38
 39def QDenseModel(weights_f, load_weights=False):
 40    """Construct QDenseModel."""
 41
 42    x = x_in = Input((RESHAPED,), name="input")
 43    x = QActivation("quantized_relu(4)", name="act_i")(x)
 44    x = QDense(
 45        N_HIDDEN,
 46        kernel_quantizer=ternary(),
 47        bias_quantizer=quantized_bits(4, 0, 1),
 48        name="dense0",
 49    )(x)
 50    x = QActivation("quantized_relu(2)", name="act0")(x)
 51    x = QDense(
 52        NB_CLASSES,
 53        kernel_quantizer=quantized_bits(4, 0, 1),
 54        bias_quantizer=quantized_bits(4, 0, 1),
 55        name="dense2",
 56    )(x)
 57    x = Activation("softmax", name="softmax")(x)
 58
 59    model = Model(inputs=[x_in], outputs=[x])
 60    model.summary()
 61    model.compile(
 62        loss="categorical_crossentropy", optimizer=OPTIMIZER, metrics=["accuracy"]
 63    )
 64
 65    if load_weights and weights_f:
 66        model.load_weights(weights_f)
 67
 68    print_qstats(model)
 69    return model
 70
 71
 72def UseNetwork(weights_f, load_weights=False):
 73    """Use DenseModel.
 74
 75    Args:
 76      weights_f: weight file location.
 77      load_weights: load weights when it is True.
 78    """
 79    model = QDenseModel(weights_f, load_weights)
 80
 81    batch_size = BATCH_SIZE
 82    (x_train_, y_train_), (x_test_, y_test_) = mnist.load_data()
 83
 84    x_train_ = x_train_.reshape(60000, RESHAPED)
 85    x_test_ = x_test_.reshape(10000, RESHAPED)
 86    x_train_ = x_train_.astype(float)
 87    x_test_ = x_test_.astype(float)
 88
 89    x_train_ /= 255
 90    x_test_ /= 255
 91
 92    print(x_train_.shape[0], "train samples")
 93    print(x_test_.shape[0], "test samples")
 94
 95    y_train_ = to_categorical(y_train_, NB_CLASSES)
 96    y_test_ = to_categorical(y_test_, NB_CLASSES)
 97
 98    if not load_weights:
 99        model.fit(
100            x_train_,
101            y_train_,
102            batch_size=batch_size,
103            epochs=NB_EPOCH,
104            verbose=VERBOSE,
105            validation_split=VALIDATION_SPLIT,
106        )
107
108        if weights_f:
109            model.save_weights(weights_f)
110
111    score = model.evaluate(x_test_, y_test_, verbose=VERBOSE)
112    print_qstats(model)
113    print("Test score:", score[0])
114    print("Test accuracy:", score[1])
115
116
117def ParserArgs():
118    parser = argparse.ArgumentParser()
119    parser.add_argument(
120        "-l",
121        "--load_weight",
122        default="0",
123        help="""load weights directly from file.
124                            0 is to disable and train the network.""",
125    )
126    parser.add_argument("-w", "--weight_file", default=None)
127    a = parser.parse_args()
128    return a
129
130
131if __name__ == "__main__":
132    args = ParserArgs()
133    lw = False if args.load_weight == "0" else True
134    UseNetwork(args.weight_file, load_weights=lw)

example_keras_to_qkeras.py

 1# Copyright 2019 Google LLC
 2#
 3#
 4# Licensed under the Apache License, Version 2.0 (the "License");
 5# you may not use this file except in compliance with the License.
 6# You may obtain a copy of the License at
 7#
 8#     http://www.apache.org/licenses/LICENSE-2.0
 9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15# ==============================================================================
16"""Tests automatic conversion of keras model to qkeras."""
17
18
19from keras.datasets import mnist
20from keras.layers import *
21from keras.models import Model
22
23from qkeras.estimate import print_qstats
24from qkeras.utils import model_quantize, quantized_model_dump
25
26x0 = x_in0 = Input((28, 28, 1), name="input0")
27x1 = x_in1 = Input((28, 28, 1), name="input1")
28x = Concatenate(name="concat")([x0, x1])
29x = Conv2D(128, (3, 3), strides=1, name="conv2d_0_m")(x)
30x = Activation("relu", name="act0_m")(x)
31x = MaxPooling2D(2, 2, name="mp_0")(x)
32x = Conv2D(256, (3, 3), strides=1, name="conv2d_1_m")(x)
33x = Activation("relu", name="act1_m")(x)
34x = MaxPooling2D(2, 2, name="mp_1")(x)
35x = Conv2D(128, (3, 3), strides=1, name="conv2d_2_m")(x)
36x = Activation("relu", name="act2_m")(x)
37x = MaxPooling2D(2, 2, name="mp_2")(x)
38x = Flatten()(x)
39x = Dense(10, name="dense")(x)
40x = Activation("softmax", name="softmax")(x)
41
42model = Model(inputs=[x_in0, x_in1], outputs=[x])
43model.summary()
44
45q_dict = {
46    "conv2d_0_m": {
47        "kernel_quantizer": "binary()",
48        "bias_quantizer": "quantized_bits(4,0,1)",
49    },
50    "conv2d_1_m": {
51        "kernel_quantizer": "ternary()",
52        "bias_quantizer": "quantized_bits(4,0,1)",
53    },
54    "act2_m": "quantized_relu(6,2)",
55    "QActivation": {"relu": "quantized_relu(4,0)"},
56    "QConv2D": {
57        "kernel_quantizer": "quantized_bits(4,0,1)",
58        "bias_quantizer": "quantized_bits(4,0,1)",
59    },
60    "QDense": {
61        "kernel_quantizer": "quantized_bits(3,0,1)",
62        "bias_quantizer": "quantized_bits(3,0,1)",
63    },
64}
65
66qmodel = model_quantize(model, q_dict, 4)
67
68qmodel.summary()
69
70print_qstats(qmodel)
71
72(x_train, y_train), (x_test, y_test) = mnist.load_data()
73
74x_test_arr = [x_test[0:10, :], x_test[0:10, :]]
75
76quantized_model_dump(
77    qmodel,
78    x_test_arr,
79    layers_to_dump=["input0", "input1", "act2_m", "act1_m", "act0_m"],
80)

example_b2t.py

 1# Copyright 2019 Google LLC
 2#
 3#
 4# Licensed under the Apache License, Version 2.0 (the "License");
 5# you may not use this file except in compliance with the License.
 6# You may obtain a copy of the License at
 7#
 8#     http://www.apache.org/licenses/LICENSE-2.0
 9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15# ==============================================================================
16"""Implements total/partial Binary to Thermometer decoder."""
17
18import numpy as np
19
20from qkeras import BinaryToThermometer
21
22if __name__ == "__main__":
23    np.random.seed(42)
24    x = np.array(range(8))
25    b = BinaryToThermometer(x, 2, 8)
26    print(b)
27    b = BinaryToThermometer(x, 2, 8, 1)
28    print(b)
29    b = BinaryToThermometer(x, 2, 8, 1, use_two_hot_encoding=1)
30    print(b)
31    b = BinaryToThermometer(x, 4, 8)
32    print(b)
33    b = BinaryToThermometer(x, 4, 8, 1)
34    print(b)
35    b = BinaryToThermometer(x, 4, 8, 1, use_two_hot_encoding=1)
36    print(b)
37    x = np.random.randint(0, 255, (100, 28, 28, 1))
38    print(x[0, 0, 0:5])
39    b = BinaryToThermometer(x, 8, 256, 0)
40    print(x.shape, b.shape)
41    print(b[0, 0, 0:5])
42    b = BinaryToThermometer(x, 8, 256, 1)
43    print(b[0, 0, 0:5])
44    x = np.random.randint(0, 255, (100, 28, 28, 2))
45    b = BinaryToThermometer(x, 8, 256, 0, 1)
46    print(x.shape, b.shape)
47    print(x[0, 0, 0, 0:2])
48    print(b[0, 0, 0, 0:8])
49    print(b[0, 0, 0, 8:16])

example_mnist_prune.py

example_mnist_ae.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""uses po2."""
 17
 18
 19
 20from keras.datasets import mnist
 21from keras.layers import *
 22from keras.layers import Activation, Input
 23from keras.models import Model
 24from keras.optimizers import Adam
 25from keras.utils import to_categorical
 26
 27from qkeras import *
 28
 29NB_EPOCH = 10
 30BATCH_SIZE = 64
 31VERBOSE = 1
 32NB_CLASSES = 10
 33OPTIMIZER = Adam(learning_rate=0.0001, decay=0.000025)
 34VALIDATION_SPLIT = 0.1
 35
 36train = 1
 37
 38(x_train, y_train), (x_test, y_test) = mnist.load_data()
 39
 40RESHAPED = 784
 41
 42x_train = x_train.astype(float)
 43x_test = x_test.astype(float)
 44
 45x_train = x_train[..., np.newaxis]
 46x_test = x_test[..., np.newaxis]
 47
 48x_train /= 256.0
 49x_test /= 256.0
 50
 51print(x_train.shape[0], "train samples")
 52print(x_test.shape[0], "test samples")
 53
 54print(y_train[0:10])
 55
 56y_train = to_categorical(y_train, NB_CLASSES)
 57y_test = to_categorical(y_test, NB_CLASSES)
 58
 59x = x_in = Input(x_train.shape[1:-1] + (1,))
 60x = QConv2D(
 61    32,
 62    kernel_size=(3, 3),
 63    kernel_quantizer=quantized_bits(4, 0, 1),
 64    bias_quantizer=quantized_bits(4, 0, 1),
 65)(x)
 66x = QActivation("quantized_relu(4,0)")(x)
 67x = QConv2D(
 68    16,
 69    kernel_size=(3, 3),
 70    kernel_quantizer=quantized_bits(4, 0, 1),
 71    bias_quantizer=quantized_bits(4, 0, 1),
 72)(x)
 73x = QActivation("quantized_relu(4,0)")(x)
 74x = QConv2D(
 75    8,
 76    kernel_size=(3, 3),
 77    kernel_quantizer=quantized_bits(4, 0, 1),
 78    bias_quantizer=quantized_bits(4, 0, 1),
 79)(x)
 80x = QActivation("quantized_relu(4,0)")(x)
 81x = QConv2DTranspose(
 82    8,
 83    kernel_size=(3, 3),
 84    kernel_quantizer=quantized_bits(4, 0, 1),
 85    bias_quantizer=quantized_bits(4, 0, 1),
 86)(x)
 87x = QActivation("quantized_relu(4,0)")(x)
 88x = QConv2DTranspose(
 89    16,
 90    kernel_size=(3, 3),
 91    kernel_quantizer=quantized_bits(4, 0, 1),
 92    bias_quantizer=quantized_bits(4, 0, 1),
 93)(x)
 94x = QActivation("quantized_relu(4,0)")(x)
 95x = QConv2DTranspose(
 96    32,
 97    kernel_size=(3, 3),
 98    kernel_quantizer=quantized_bits(4, 0, 1),
 99    bias_quantizer=quantized_bits(4, 0, 1),
100)(x)
101x = QActivation("quantized_relu(4,0)")(x)
102x = QConv2D(
103    1,
104    kernel_size=(3, 3),
105    padding="same",
106    kernel_quantizer=quantized_bits(4, 0, 1),
107    bias_quantizer=quantized_bits(4, 0, 1),
108)(x)
109x_out = x
110x = Activation("sigmoid")(x)
111
112model = Model(inputs=[x_in], outputs=[x])
113mo = Model(inputs=[x_in], outputs=[x_out])
114model.summary()
115
116model.compile(loss="binary_crossentropy", optimizer=OPTIMIZER, metrics=["accuracy"])
117
118if train:
119    history = model.fit(
120        x_train,
121        x_train,
122        batch_size=BATCH_SIZE,
123        epochs=NB_EPOCH,
124        initial_epoch=1,
125        verbose=VERBOSE,
126        validation_split=VALIDATION_SPLIT,
127    )
128
129    # Generate reconstructions
130    num_reco = 8
131    samples = x_test[:num_reco]
132    targets = y_test[:num_reco]
133    reconstructions = model.predict(samples)
134
135
136for layer in model.layers:
137    for w, weight in enumerate(layer.get_weights()):
138        print(layer.name, w, weight.shape)
139
140print_qstats(model)

example_mnist_bn.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""Tests mnist batchnormalization used as learned scale factor."""
 17
 18# to run, THRESHOLD=0.05 WITH_BN=1 EPOCHS=5 TRAIN=1 python example_mnist_bn.py
 19
 20
 21
 22import keras.backend as K
 23import keras.ops.numpy as knp
 24from keras import callbacks
 25from keras.datasets import mnist
 26from keras.layers import *
 27from keras.models import Model
 28from keras.optimizers import *
 29from keras.utils import to_categorical
 30from six.moves import zip
 31
 32from qkeras import *
 33
 34TRAIN = 1
 35NB_EPOCH = 2
 36BATCH_SIZE = 64
 37VERBOSE = 1
 38NB_CLASSES = 10
 39OPTIMIZER = Adam(learning_rate=0.0001)
 40VALIDATION_SPLIT = 0.1
 41WITH_BN = 1
 42THRESHOLD = 0.1
 43
 44
 45class LearningRateAdjuster(callbacks.Callback):
 46    def __init__(self):
 47        self.learning_rate_factor = 1.0
 48        pass
 49
 50    def on_epoch_end(self, epochs, logs):
 51        max_variance = -1
 52
 53        for layer in self.model.layers:
 54            if layer.__class__.__name__ in [
 55                "BatchNormalization",
 56                "QBatchNormalization",
 57            ]:
 58                variance = knp.max(layer.get_weights()[-1])
 59                max_variance = max(variance, max_variance)
 60
 61        if max_variance > 32 and self.learning_rate_factor < 100:
 62            learning_rate = K.get_value(self.model.optimizer.learning_rate)
 63            self.learning_rate_factor /= 2.0
 64            print(
 65                f"***** max_variance is {max_variance} / lr is {learning_rate} *****"
 66            )
 67            K.eval(K.update(self.model.optimizer.learning_rate, learning_rate / 2.0))
 68
 69
 70lra = LearningRateAdjuster()
 71
 72(x_train, y_train), (x_test, y_test) = mnist.load_data()
 73
 74x_train = x_train.reshape(x_train.shape + (1,)).astype(float)
 75x_test = x_test.reshape(x_test.shape + (1,)).astype(float)
 76
 77x_train /= 256.0
 78x_test /= 256.0
 79
 80print(x_train.shape[0], "train samples")
 81print(x_test.shape[0], "test samples")
 82
 83print(y_train[0:10])
 84
 85y_train = to_categorical(y_train, NB_CLASSES)
 86y_test = to_categorical(y_test, NB_CLASSES)
 87
 88x = x_in = Input(x_train.shape[1:], name="input")
 89# x = QActivation("quantized_relu_po2(4,1)", name="acti")(x)
 90x = QConv2D(
 91    128,
 92    (3, 3),
 93    strides=1,
 94    kernel_quantizer=ternary(),  # quantized_po2(4, 1),
 95    bias_quantizer=quantized_bits(4, 2, 0) if not WITH_BN else None,
 96    bias_range=4 if not WITH_BN else None,
 97    use_bias=not WITH_BN,
 98    name="conv2d_0_m",
 99)(x)
100if WITH_BN:
101    x = QBatchNormalization(
102        gamma_quantizer=quantized_relu_po2(4, 8),
103        variance_quantizer=quantized_relu_po2(6),
104        beta_quantizer=quantized_po2(4, 4),
105        gamma_range=8,
106        beta_range=4,
107        name="bn0",
108    )(x)
109x = QActivation("quantized_relu(3,1)", name="act0_m")(x)
110x = MaxPooling2D(2, 2, name="mp_0")(x)
111x = QConv2D(
112    256,
113    (3, 3),
114    strides=1,
115    kernel_quantizer=ternary(),  # quantized_bits(2,0,1),
116    bias_quantizer=quantized_bits(4, 2, 1) if not WITH_BN else None,
117    bias_range=4 if not WITH_BN else None,
118    use_bias=not WITH_BN,
119    name="conv2d_1_m",
120)(x)
121if WITH_BN:
122    x = QBatchNormalization(
123        gamma_quantizer=quantized_relu_po2(4, 8),
124        variance_quantizer=quantized_relu_po2(6),
125        beta_quantizer=quantized_po2(4, 4),
126        gamma_range=8,
127        beta_range=4,
128        name="bn1",
129    )(x)
130x = QActivation("quantized_relu(3,1)", name="act1_m")(x)
131x = MaxPooling2D(2, 2, name="mp_1")(x)
132x = QConv2D(
133    128,
134    (3, 3),
135    strides=1,
136    kernel_quantizer=ternary(),  # quantized_bits(2,0,1),
137    bias_quantizer=quantized_bits(4, 2, 1) if not WITH_BN else None,
138    bias_range=4 if not WITH_BN else None,
139    use_bias=not WITH_BN,
140    name="conv2d_2_m",
141)(x)
142if WITH_BN:
143    x = QBatchNormalization(
144        gamma_quantizer=quantized_relu_po2(4, 8),
145        variance_quantizer=quantized_relu_po2(6),
146        beta_quantizer=quantized_po2(4, 4),
147        gamma_range=8,
148        beta_range=4,
149        name="bn2",
150    )(x)
151x = QActivation("quantized_relu(3,1)", name="act2_m")(x)
152x = MaxPooling2D(2, 2, name="mp_2")(x)
153x = Flatten()(x)
154x = QDense(
155    NB_CLASSES,
156    kernel_quantizer=quantized_ulaw(4, 0, 1),
157    bias_quantizer=quantized_bits(4, 0, 1),
158    name="dense",
159)(x)
160x = Activation("softmax", name="softmax")(x)
161
162model = Model(inputs=[x_in], outputs=[x])
163model.summary()
164
165model.compile(
166    loss="categorical_crossentropy", optimizer=OPTIMIZER, metrics=["accuracy"]
167)
168
169
170if TRAIN:
171    history = model.fit(
172        x_train,
173        y_train,
174        batch_size=BATCH_SIZE,
175        epochs=NB_EPOCH,
176        initial_epoch=1,
177        verbose=VERBOSE,
178        validation_split=VALIDATION_SPLIT,
179        callbacks=[],
180    )  # lra])
181
182    outputs = []
183    output_names = []
184
185    for layer in model.layers:
186        if layer.__class__.__name__ in [
187            "QActivation",
188            "QBatchNormalization",
189            "Activation",
190            "QDense",
191            "QConv2D",
192            "QDepthwiseConv2D",
193        ]:
194            output_names.append(layer.name)
195            outputs.append(layer.output)
196
197    model_debug = Model(inputs=[x_in], outputs=outputs)
198
199    outputs = model_debug.predict(x_train)
200
201    print("{:30} {: 8.4f} {: 8.4f}".format("input", knp.min(x_train), knp.max(x_train)))
202
203    for n, p in zip(output_names, outputs):
204        print(f"{n:30} {knp.min(p): 8.4f} {knp.max(p): 8.4f}", end="")
205        layer = model.get_layer(n)
206        for i, weights in enumerate(layer.get_weights()):
207            if layer.get_quantizers()[i]:
208                weights = K.eval(layer.get_quantizers()[i](K.constant(weights)))
209            print(
210                f" ({knp.min(weights): 8.4f} {knp.max(weights): 8.4f})", end=""
211            )
212        print("")
213
214    score = model.evaluate(x_test, y_test, verbose=False)
215    print("Test score:", score[0])
216    print("Test accuracy:", score[1])
217
218print_qstats(model)

example_mnist_b2t.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""Tests qcore model with BinaryToThermometer."""
 17
 18
 19import os
 20
 21import keras.ops.numpy as knp
 22from keras.datasets import mnist
 23from keras.layers import *
 24from keras.layers import Activation, Flatten, Input
 25from keras.models import Model
 26from keras.optimizers import Adam
 27from keras.utils import to_categorical
 28
 29from qkeras import *
 30
 31NB_EPOCH = 20
 32BATCH_SIZE = 32
 33VERBOSE = 1
 34NB_CLASSES = 10
 35OPTIMIZER = Adam(learning_rate=0.0001)
 36N_HIDDEN = 100
 37VALIDATION_SPLIT = 0.1
 38
 39T_CLASSES = 256
 40T_WITH_RESIDUE = 0
 41
 42(x_train, y_train), (x_test, y_test) = mnist.load_data()
 43
 44RESHAPED = 784
 45
 46x_train = x_train.astype(float)
 47x_test = x_test.astype(float)
 48
 49x_train = x_train[..., np.newaxis]
 50x_test = x_test[..., np.newaxis]
 51
 52if T_CLASSES == 1:
 53    x_train /= 256.0
 54    x_test /= 256.0
 55
 56print(x_train.shape[0], "train samples")
 57print(x_test.shape[0], "test samples")
 58
 59print(y_train[0:10])
 60
 61# x_train = x_train[0:1000]
 62# y_train = y_train[0:1000]
 63# x_test = x_test[0:100]
 64# y_test = y_test[0:100]
 65
 66y_train = to_categorical(y_train, NB_CLASSES)
 67y_test = to_categorical(y_test, NB_CLASSES)
 68
 69# we ran out of memory here, so we split x_train/x_test into smaller groups
 70
 71x = x_in = Input(x_train.shape[1:-1] + (T_CLASSES,), name="input")
 72
 73# Number is represented as 1.bbb, where number of bits of bbb is
 74# log2(256/T_CLASSES) if T_WITH_RESIDUE == 1
 75
 76bits = (T_WITH_RESIDUE == 1) * int(knp.ceil(knp.log2(256 / T_CLASSES))) + (T_CLASSES > 1)
 77
 78print(f"Input quantizer: quantized_relu({bits},{int(T_CLASSES > 1)})")
 79x = QActivation(f"quantized_relu({bits},{int(T_CLASSES > 1)})")(x)
 80x = QConv2D(
 81    64,
 82    (3, 3),
 83    strides=1,
 84    padding="same",
 85    kernel_quantizer=quantized_po2(4, 1),
 86    bias_quantizer=quantized_bits(4, 2, 1),
 87    bias_range=4,
 88    name="conv2d_0_m",
 89)(x)
 90x = QActivation("quantized_relu(4,0)", name="act0_m")(x)
 91x = MaxPooling2D(2, 2, name="mp_0")(x)
 92x = QConv2D(
 93    32,
 94    (3, 3),
 95    strides=1,
 96    padding="same",
 97    kernel_quantizer=stochastic_ternary(),
 98    bias_quantizer=quantized_bits(8, 5, 1),
 99    bias_range=32,
100    name="conv2d_1_m",
101)(x)
102x = QActivation("quantized_relu(4,0)", name="act1_m")(x)
103x = MaxPooling2D(2, 2, name="mp_1")(x)
104x = QConv2D(
105    16,
106    (3, 3),
107    strides=1,
108    padding="same",
109    kernel_quantizer=quantized_bits(4, 0, 1),
110    bias_quantizer=quantized_bits(8, 5, 1),
111    bias_range=32,
112    name="conv2d_2_m",
113)(x)
114x = QActivation("quantized_relu(6,2)", name="act2_m")(x)
115x = MaxPooling2D(2, 2, name="mp_2")(x)
116x = Flatten()(x)
117x = QDense(
118    NB_CLASSES,
119    kernel_quantizer=quantized_bits(4, 0, 1),
120    bias_quantizer=quantized_bits(4, 0, 1),
121    name="dense2",
122)(x)
123x = Activation("softmax", name="softmax")(x)
124
125model = Model(inputs=[x_in], outputs=[x])
126model.summary()
127
128model.compile(
129    loss="categorical_crossentropy", optimizer=OPTIMIZER, metrics=["accuracy"]
130)
131
132outputs = []
133output_names = []
134
135for layer in model.layers:
136    if layer.__class__.__name__ in [
137        "QActivation",
138        "Activation",
139        "QDense",
140        "QConv2D",
141        "QDepthwiseConv2D",
142    ]:
143        output_names.append(layer.name)
144        outputs.append(layer.output)
145
146model_debug = Model(inputs=[x_in], outputs=outputs)
147
148batch_size = 1000 * BATCH_SIZE
149n_batches = x_train.shape[0] // batch_size
150
151if T_CLASSES > 1:
152    x_test = BinaryToThermometer(x_test, T_CLASSES, 256, T_WITH_RESIDUE)
153
154if int(os.environ.get("TRAIN", 0)):
155    for i in range(NB_EPOCH):
156        for b in range(n_batches):
157            min_b = b * batch_size
158            max_b = (b + 1) * batch_size
159            max_b = min(max_b, x_train.shape[0])
160
161            if T_CLASSES > 1:
162                x = BinaryToThermometer(
163                    x_train[min_b:max_b], T_CLASSES, 256, T_WITH_RESIDUE
164                )
165            else:
166                x = x_train[min_b:max_b]
167
168            history = model.fit(
169                x,
170                y_train[min_b:max_b],
171                batch_size=BATCH_SIZE,
172                epochs=i + 1,
173                initial_epoch=i,
174                verbose=VERBOSE,
175                validation_split=VALIDATION_SPLIT,
176            )
177
178    if T_CLASSES > 1:
179        x = BinaryToThermometer(x_train[0:100], T_CLASSES, 256, T_WITH_RESIDUE)
180    else:
181        x = x_train[0:100]
182
183    outputs = model_debug.predict(x)
184
185    print("{:30} {: 8.4f} {: 8.4f}".format("input", knp.min(x), knp.max(x)))
186    for n, p in zip(output_names, outputs):
187        print(f"{n:30} {knp.min(p): 8.4f} {knp.max(p): 8.4f}", end="")
188        layer = model.get_layer(n)
189        for i, weights in enumerate(layer.get_weights()):
190            weights = layer.get_quantizers()[i](weights)
191            print(
192                f" ({knp.min(weights): 8.4f} {knp.max(weights): 8.4f})", end=""
193            )
194        print("")
195
196    score = model.evaluate(x_test, y_test, verbose=VERBOSE)
197    print("Test score:", score[0])
198    print("Test accuracy:", score[1])
199
200print_qstats(model)
201
202acc = analyze_accumulator_from_sample(model, x_test, mode="sampled")
203
204print(acc)

example_mnist_po2.py

  1# Copyright 2019 Google LLC
  2#
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8#     http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15# ==============================================================================
 16"""Tests qlayers model with po2."""
 17
 18
 19import keras.backend as K
 20import keras.ops.numpy as knp
 21from keras.datasets import mnist
 22from keras.layers import Activation, Flatten, Input
 23from keras.models import Model
 24from keras.optimizers import Adam
 25from keras.utils import to_categorical
 26
 27from qkeras import *  # pylint: disable=wildcard-import
 28
 29NB_EPOCH = 5
 30BATCH_SIZE = 64
 31VERBOSE = 1
 32NB_CLASSES = 10
 33OPTIMIZER = Adam(learning_rate=0.0001, decay=0.000025)
 34N_HIDDEN = 100
 35VALIDATION_SPLIT = 0.1
 36
 37QUANTIZED = 1
 38CONV2D = 1
 39
 40(x_train, y_train), (x_test, y_test) = mnist.load_data()
 41
 42RESHAPED = 784
 43
 44x_train = x_train.astype(float)
 45x_test = x_test.astype(float)
 46
 47x_train = x_train[..., np.newaxis]
 48x_test = x_test[..., np.newaxis]
 49
 50x_train /= 256.0
 51x_test /= 256.0
 52
 53train = False
 54
 55print(x_train.shape[0], "train samples")
 56print(x_test.shape[0], "test samples")
 57
 58print(y_train[0:10])
 59
 60y_train = to_categorical(y_train, NB_CLASSES)
 61y_test = to_categorical(y_test, NB_CLASSES)
 62
 63# we ran out of memory here, so we split x_train/x_test into smaller groups
 64
 65x = x_in = Input(x_train.shape[1:-1] + (1,), name="input")
 66x = QActivation("quantized_relu_po2(4)", name="acti")(x)
 67x = QConv2D(
 68    32,
 69    (2, 2),
 70    strides=(2, 2),
 71    kernel_quantizer=quantized_po2(4, 1),
 72    bias_quantizer=quantized_po2(4, 1),
 73    name="conv2d_0_m",
 74)(x)
 75x = QActivation("quantized_relu_po2(4,4)", name="act0_m")(x)
 76x = QConv2D(
 77    64,
 78    (3, 3),
 79    strides=(2, 2),
 80    kernel_quantizer=quantized_po2(4, 1),
 81    bias_quantizer=quantized_po2(4, 1),
 82    name="conv2d_1_m",
 83)(x)
 84x = QActivation("quantized_relu_po2(4,4,use_stochastic_rounding=True)", name="act1_m")(
 85    x
 86)
 87x = QConv2D(
 88    64,
 89    (2, 2),
 90    strides=(2, 2),
 91    kernel_quantizer=quantized_po2(4, 1, use_stochastic_rounding=True),
 92    bias_quantizer=quantized_po2(4, 1),
 93    name="conv2d_2_m",
 94)(x)
 95x = QActivation("quantized_relu(4,1)", name="act2_m")(x)
 96x = Flatten()(x)
 97x = QDense(
 98    NB_CLASSES,
 99    kernel_quantizer=quantized_bits(4, 0, 1),
100    bias_quantizer=quantized_bits(4, 0, 1),
101    name="dense",
102)(x)
103x = Activation("softmax", name="softmax")(x)
104
105model = Model(inputs=[x_in], outputs=[x])
106model.summary()
107
108model.compile(
109    loss="categorical_crossentropy", optimizer=OPTIMIZER, metrics=["accuracy"]
110)
111
112if train:
113    history = model.fit(
114        x_train,
115        y_train,
116        batch_size=BATCH_SIZE,
117        epochs=NB_EPOCH,
118        initial_epoch=1,
119        verbose=VERBOSE,
120        validation_split=VALIDATION_SPLIT,
121    )
122
123    outputs = []
124    output_names = []
125
126    for layer in model.layers:
127        if layer.__class__.__name__ in [
128            "QActivation",
129            "Activation",
130            "QDense",
131            "QConv2D",
132            "QDepthwiseConv2D",
133        ]:
134            output_names.append(layer.name)
135            outputs.append(layer.output)
136
137    model_debug = Model(inputs=[x_in], outputs=outputs)
138
139    outputs = model_debug.predict(x_train)
140
141    print("{:30} {: 8.4f} {: 8.4f}".format("input", knp.min(x_train), knp.max(x_train)))
142
143    for n, p in zip(output_names, outputs):
144        print(f"{n:30} {knp.min(p): 8.4f} {knp.max(p): 8.4f}", end="")
145        layer = model.get_layer(n)
146        for i, weights in enumerate(layer.get_weights()):
147            weights = K.eval(layer.get_quantizers()[i](K.constant(weights)))
148            print(
149                f" ({knp.min(weights): 8.4f} {knp.max(weights): 8.4f})", end=""
150            )
151            print("")
152
153    score = model.evaluate(x_test, y_test, verbose=VERBOSE)
154    print("Test score:", score[0])
155    print("Test accuracy:", score[1])
156
157model.summary()
158
159print_qstats(model)