Tensorflow技術點整理

語言: CN / TW / HK

時間 2021-10-23 05:01:49 演算法之名

主題: keras def fashion-mnist numpy layer matplotlib

我們用tensorflow2來看一個服飾資料集的識別。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    print(x_valid.shape, y_valid.shape)
    print(x_train.shape, y_train.shape)
    print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    show_single_image(x_train[1])

執行結果

(5000, 28, 28) (5000,)
(55000, 28, 28) (55000,)
(10000, 28, 28) (10000,)

這裡我們可以看到，這些影象資料集都是28*28的單通道圖片。我們將這些資料集分成了55000個訓練資料集，5000個驗證資料集以及10000個測試資料集。現在我們再來看一下這些資料集的分類型別

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    show_imgs(3, 5, x_train, y_train, class_names)

執行結果

這裡我們可以看到這些服飾總共有10種類型。現在我們就使用神經網路來對這些資料進行訓練和驗證

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵損失函式，隨機梯度下降法進行訓練
    # 這裡需要說明的是如果是categorical_crossentropy
    # 則需要將標籤資料集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    history = model.fit(x_train, y_train, epochs=10, validation_data=(x_valid, y_valid))
    print(model.summary())
    print(history.history)

    def plot_learning_curves(history):
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
55000/55000 [==============================] - 6s 107us/sample - loss: 2.2443 - accuracy: 0.7192 - val_loss: 0.6124 - val_accuracy: 0.7928
Epoch 2/10
55000/55000 [==============================] - 5s 95us/sample - loss: 0.5678 - accuracy: 0.7980 - val_loss: 0.5467 - val_accuracy: 0.8018
Epoch 3/10
55000/55000 [==============================] - 5s 92us/sample - loss: 0.4991 - accuracy: 0.8188 - val_loss: 0.4968 - val_accuracy: 0.8358
Epoch 4/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.4646 - accuracy: 0.8316 - val_loss: 0.4910 - val_accuracy: 0.8332
Epoch 5/10
55000/55000 [==============================] - 7s 119us/sample - loss: 0.4371 - accuracy: 0.8393 - val_loss: 0.4768 - val_accuracy: 0.8374
Epoch 6/10
55000/55000 [==============================] - 7s 123us/sample - loss: 0.4211 - accuracy: 0.8455 - val_loss: 0.4608 - val_accuracy: 0.8444
Epoch 7/10
55000/55000 [==============================] - 6s 103us/sample - loss: 0.4046 - accuracy: 0.8504 - val_loss: 0.4506 - val_accuracy: 0.8464
Epoch 8/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.3902 - accuracy: 0.8550 - val_loss: 0.4427 - val_accuracy: 0.8480
Epoch 9/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.3796 - accuracy: 0.8581 - val_loss: 0.4814 - val_accuracy: 0.8312
Epoch 10/10
55000/55000 [==============================] - 5s 99us/sample - loss: 0.3714 - accuracy: 0.8608 - val_loss: 0.4665 - val_accuracy: 0.8440
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 266,610
Trainable params: 266,610
Non-trainable params: 0
_________________________________________________________________
None
{'loss': [2.2442730520161716, 0.5677844335772775, 0.49907195046598263, 0.46455069565122775, 0.43706936968456617, 0.4210625493981621, 0.4045901944290508, 0.3901773897236044, 0.3795636681730097, 0.37135513420971955], 'accuracy': [0.71916366, 0.7979818, 0.8187636, 0.83156365, 0.8393091, 0.84549093, 0.8503636, 0.8550182, 0.8580545, 0.8608364], 'val_loss': [0.6123501856803895, 0.5466709545612335, 0.4968059474229813, 0.49100303683280944, 0.4768493502855301, 0.46080493590831756, 0.4505628205776215, 0.44266556935310364, 0.4814212405562401, 0.4664571787118912], 'val_accuracy': [0.7928, 0.8018, 0.8358, 0.8332, 0.8374, 0.8444, 0.8464, 0.848, 0.8312, 0.844]}

這裡我們使用了三個全連線層來構建神經網路，對驗證資料集的準確率達到了85.92%。這裡我們使用的是隨機梯度下降法，關於隨機梯度下降法的內容請參考機器學習演算法整理中的隨機梯度下降法。下面這個報告是告訴我們在神經網路中每一個全連線層的維度以及引數量。由於我們的影象是28*28，所以扁平化處理後是28*28=784個維度。經過全連線層之後，我們要把第一個全連線層的輸出變成樣本數*300個維度的矩陣，我們需要將矩陣[樣本數，784]•[784, 300]的矩陣W(有關矩陣乘法的內容請參考線性代數整理中矩陣和矩陣的乘法)，一般我們會加上一個b，就是一個長度為300的向量。這樣就得到了一個[樣本數, 300]的輸出矩陣。而引數量就等於784*300+300=235500。同理，第二個全連線層的輸出為樣本數*100的矩陣，我們需要將第一層的輸出矩陣[樣本數, 300]•[300, 100]+b，就得到[樣本數, 100]的輸出，引數量即為300*100+100=30100。而圖中也顯示出損失函式值是在穩固下降的，而識別準確率是在穩固上升的。現在我們來給訓練資料加上歸一化處理。我們先來看一下歸一化前後訓練資料集的最大值和最小值

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片資料轉成二維資料[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片資料,這裡fit_transform會記錄訓練資料集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裡驗證資料集會採用訓練資料集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    print(np.max(x_train), np.min(x_train))

執行結果

255 0
2.0231433 -0.8105136

這裡我們可以看到，訓練資料集歸一化前後它們的最大值最小值是不同的。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片資料轉成二維資料[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片資料,這裡fit_transform會記錄訓練資料集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裡驗證資料集會採用訓練資料集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵損失函式，隨機梯度下降法進行訓練
    # 這裡需要說明的是如果是categorical_crossentropy
    # 則需要將標籤資料集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    history = model.fit(x_train, y_train, epochs=10, validation_data=(x_valid, y_valid))
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    print(history.history)

    def plot_learning_curves(history):
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
55000/55000 [==============================] - 5s 95us/sample - loss: 0.9122 - accuracy: 0.7003 - val_loss: 0.6157 - val_accuracy: 0.7920
Epoch 2/10
55000/55000 [==============================] - 5s 83us/sample - loss: 0.5771 - accuracy: 0.8010 - val_loss: 0.5165 - val_accuracy: 0.8182
Epoch 3/10
55000/55000 [==============================] - 6s 104us/sample - loss: 0.5075 - accuracy: 0.8235 - val_loss: 0.4732 - val_accuracy: 0.8378
Epoch 4/10
55000/55000 [==============================] - 7s 127us/sample - loss: 0.4716 - accuracy: 0.8343 - val_loss: 0.4473 - val_accuracy: 0.8480
Epoch 5/10
55000/55000 [==============================] - 6s 117us/sample - loss: 0.4477 - accuracy: 0.8421 - val_loss: 0.4323 - val_accuracy: 0.8530
Epoch 6/10
55000/55000 [==============================] - 6s 110us/sample - loss: 0.4306 - accuracy: 0.8483 - val_loss: 0.4175 - val_accuracy: 0.8580
Epoch 7/10
55000/55000 [==============================] - 5s 90us/sample - loss: 0.4167 - accuracy: 0.8521 - val_loss: 0.4101 - val_accuracy: 0.8564
Epoch 8/10
55000/55000 [==============================] - 5s 90us/sample - loss: 0.4059 - accuracy: 0.8562 - val_loss: 0.4022 - val_accuracy: 0.8626
Epoch 9/10
55000/55000 [==============================] - 5s 92us/sample - loss: 0.3964 - accuracy: 0.8599 - val_loss: 0.3944 - val_accuracy: 0.8634
Epoch 10/10
55000/55000 [==============================] - 5s 93us/sample - loss: 0.3880 - accuracy: 0.8630 - val_loss: 0.3889 - val_accuracy: 0.8606
10000/1 ==============================================] - 1s 51us/sample - loss: 0.2991 - accuracy: 0.8472
[0.43079993045330045, 0.8472]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 266,610
Trainable params: 266,610
Non-trainable params: 0
_________________________________________________________________
None
{'loss': [0.912204550890489, 0.5771390253067017, 0.507524390927228, 0.4716112705404108, 0.4477458443338221, 0.4306134765581651, 0.416733508162065, 0.4059152506828308, 0.3963754140810533, 0.38801397878473454], 'accuracy': [0.70034546, 0.801, 0.82345456, 0.8343091, 0.84214544, 0.84827274, 0.85214543, 0.85623634, 0.8598727, 0.8629818], 'val_loss': [0.615698953294754, 0.516535887670517, 0.47322214045524597, 0.4472690547943115, 0.43233813087940215, 0.41751169788837433, 0.41005783474445345, 0.40220267939567567, 0.3943846334218979, 0.38885269825458524], 'val_accuracy': [0.792, 0.8182, 0.8378, 0.848, 0.853, 0.858, 0.8564, 0.8626, 0.8634, 0.8606]}

這裡我們對測試資料集進行了測試，它的分類準確率達到了84.72%。

回撥函式

回撥函式是當我們在訓練模型的時候，中間可能要做一些事情。比如說模型訓練中，當損失函式值loss不再下降的時候，我們可以提前停止訓練。又比如在訓練的過程中，每隔段時間把模型引數給儲存下來。又或者在模型訓練過程中，就對一些模型指標進行圖形化輸出等等。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片資料轉成二維資料[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片資料,這裡fit_transform會記錄訓練資料集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裡驗證資料集會採用訓練資料集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵損失函式，隨機梯度下降法進行訓練
    # 這裡需要說明的是如果是categorical_crossentropy
    # 則需要將標籤資料集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我們這裡使用三個回撥函式Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以儲存我們在訓練中的指標，比如損失函式值以及識別準確率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是儲存模型引數
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前終止訓練
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=10,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

當我們訓練完成之後，可以到callbacks上一層資料夾中，展開callbacks資料夾，我們可以看到這樣一些檔案

在callbacks上一層資料夾中輸入命令

(base) -bash-3.2$ tensorboard --logdir=callbacks

會得到這樣一些輸出

Serving TensorBoard on localhost; to expose to the network, use a proxy or pass --bind_all
TensorBoard 2.0.0 at http://localhost:6006/ (Press CTRL+C to quit)

此時開啟瀏覽器訪問http://localhost:6006，會看到我們訓練中的指標資料圖

以及神經網路的層級圖

而EarlyStopping的引數意義如下：

min_delta：增大或減小的閾值，只有大於這個部分才算作improvement(改善)。這個值的大小取決於monitor，也反映了你的容忍程度。例如monitor(監控)是’acc’，同時其變化範圍在70%-90%之間，所以對於小於0.01%的變化不關心。加上觀察到訓練過程中存在抖動的情況（即先下降後上升），所以適當增大容忍程度，最終設為0.001%。
patience：能夠容忍多少個epoch內都沒有improvement。這個設定其實是在抖動和真正的準確率下降之間做tradeoff(權衡)。如果patience設的大，那麼最終得到的準確率要略低於模型可以達到的最高準確率。如果patience設的小，那麼模型很可能在前期抖動，還在全圖搜尋的階段就停止了，準確率一般很差。patience的大小和learning rate直接相關。在learning rate設定的情況下，前期先訓練幾次觀察抖動的epoch number，比其稍大些設定patience。在learning rate變化的情況下，建議要略小於最大的抖動epoch number。筆者在引入EarlyStopping之前就已經得到可以接受的結果了，EarlyStopping算是錦上添花，所以patience設的比較高，設為抖動epoch number的最大值。
min_delta和patience都和“避免模型停止在抖動過程中”有關係，所以調節的時候需要互相協調。通常情況下，min_delta降低，那麼patience可以適當減少；min_delta增加，那麼patience需要適當延長；反之亦然。

歸一化與批歸一化

歸一化

Min-max歸一化：x*=(x-min)/(max-min)
Z-score歸一化：x*=(x-μ)/δ，這裡μ表示均值，δ表示方差，是我們經常使用的歸一化

批歸一化

批歸一化是指把歸一化這種思路從輸入資料上給擴充套件到網路的每層的啟用值上，每層的啟用值都做歸一化。我們知道對於神經網路來說是一種層級結構，每一層的輸出就是下一層的輸入，我們在每一層的輸入上都去做歸一化，那麼就會使得整個的網路的效果會更好。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片資料轉成二維資料[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片資料,這裡fit_transform會記錄訓練資料集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裡驗證資料集會採用訓練資料集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        # 批歸一化
        keras.layers.BatchNormalization(),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.BatchNormalization(),
        keras.layers.Dense(10, activation='softmax')
        # 上面是將批歸一化放在啟用函式之後，也可以放在啟用函式之前
        # keras.layers.Dense(300),
        # keras.layers.BatchNormalization(),
        # keras.layers.Activation('relu'),
    ])
    # 使用交叉熵損失函式，隨機梯度下降法進行訓練
    # 這裡需要說明的是如果是categorical_crossentropy
    # 則需要將標籤資料集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我們這裡使用三個回撥函式Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以儲存我們在訓練中的指標，比如損失函式值以及識別準確率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是儲存模型引數
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前終止訓練
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=10,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

執行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
   32/55000 [..............................] - ETA: 44:06 - loss: 3.0657 - accuracy: 0.06252021-10-20 23:21:46.915226: I tensorflow/core/profiler/lib/profiler_session.cc:184] Profiler session started.
55000/55000 [==============================] - 12s 223us/sample - loss: 0.8149 - accuracy: 0.7291 - val_loss: 0.5425 - val_accuracy: 0.8172
Epoch 2/10
55000/55000 [==============================] - 11s 195us/sample - loss: 0.5600 - accuracy: 0.8085 - val_loss: 0.4776 - val_accuracy: 0.8398
Epoch 3/10
55000/55000 [==============================] - 13s 229us/sample - loss: 0.5023 - accuracy: 0.8254 - val_loss: 0.4405 - val_accuracy: 0.8514
Epoch 4/10
55000/55000 [==============================] - 12s 216us/sample - loss: 0.4684 - accuracy: 0.8367 - val_loss: 0.4181 - val_accuracy: 0.8600
Epoch 5/10
55000/55000 [==============================] - 11s 200us/sample - loss: 0.4447 - accuracy: 0.8459 - val_loss: 0.4045 - val_accuracy: 0.8634
Epoch 6/10
55000/55000 [==============================] - 11s 200us/sample - loss: 0.4250 - accuracy: 0.8526 - val_loss: 0.3916 - val_accuracy: 0.8688
Epoch 7/10
55000/55000 [==============================] - 11s 204us/sample - loss: 0.4093 - accuracy: 0.8561 - val_loss: 0.3838 - val_accuracy: 0.8692
Epoch 8/10
55000/55000 [==============================] - 11s 207us/sample - loss: 0.3983 - accuracy: 0.8600 - val_loss: 0.3772 - val_accuracy: 0.8718
Epoch 9/10
55000/55000 [==============================] - 11s 207us/sample - loss: 0.3886 - accuracy: 0.8645 - val_loss: 0.3687 - val_accuracy: 0.8732
Epoch 10/10
55000/55000 [==============================] - 12s 212us/sample - loss: 0.3777 - accuracy: 0.8681 - val_loss: 0.3646 - val_accuracy: 0.8742
10000/1 [==================================================] - 1s 99us/sample - loss: 0.3293 - accuracy: 0.8609
[0.39349488220214845, 0.8609]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
batch_normalization (BatchNo (None, 300)               1200      
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
batch_normalization_1 (Batch (None, 100)               400       
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 268,210
Trainable params: 267,410
Non-trainable params: 800
_________________________________________________________________
None

通過批過一化後，我們可以看到它的識別準確率比之前都得到了提升。

dropout

有關dropout的詳細內容請參考Tensorflow深度學習演算法整理

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片資料轉成二維資料[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片資料,這裡fit_transform會記錄訓練資料集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裡驗證資料集會採用訓練資料集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        # 批歸一化
        keras.layers.BatchNormalization(),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.BatchNormalization(),
        # AlphaDropout，1均值和方差不變，2啟用值歸一化性質不變
        keras.layers.AlphaDropout(rate=0.5),
        # keras.layers.Dropout(rate=0.5),
        keras.layers.Dense(10, activation='softmax')
        # 上面是將批歸一化放在啟用函式之後，也可以放在啟用函式之前
        # keras.layers.Dense(300),
        # keras.layers.BatchNormalization(),
        # keras.layers.Activation('relu'),
    ])
    # 使用交叉熵損失函式，隨機梯度下降法進行訓練
    # 這裡需要說明的是如果是categorical_crossentropy
    # 則需要將標籤資料集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我們這裡使用三個回撥函式Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以儲存我們在訓練中的指標，比如損失函式值以及識別準確率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是儲存模型引數
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前終止訓練
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=20,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

執行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/20
   32/55000 [..............................] - ETA: 32:30 - loss: 2.7970 - accuracy: 0.18752021-10-20 23:56:36.685505: I tensorflow/core/profiler/lib/profiler_session.cc:184] Profiler session started.
55000/55000 [==============================] - 14s 252us/sample - loss: 1.6393 - accuracy: 0.4541 - val_loss: 0.7376 - val_accuracy: 0.7648
Epoch 2/20
55000/55000 [==============================] - 12s 222us/sample - loss: 1.1364 - accuracy: 0.6211 - val_loss: 0.6595 - val_accuracy: 0.7934
Epoch 3/20
55000/55000 [==============================] - 15s 275us/sample - loss: 0.9986 - accuracy: 0.6656 - val_loss: 0.6102 - val_accuracy: 0.8088
Epoch 4/20
55000/55000 [==============================] - 14s 257us/sample - loss: 0.9116 - accuracy: 0.6943 - val_loss: 0.5842 - val_accuracy: 0.8180
Epoch 5/20
55000/55000 [==============================] - 13s 232us/sample - loss: 0.8560 - accuracy: 0.7124 - val_loss: 0.5690 - val_accuracy: 0.8248
Epoch 6/20
55000/55000 [==============================] - 12s 227us/sample - loss: 0.8100 - accuracy: 0.7288 - val_loss: 0.5518 - val_accuracy: 0.8328
Epoch 7/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.7779 - accuracy: 0.7401 - val_loss: 0.5428 - val_accuracy: 0.8372
Epoch 8/20
55000/55000 [==============================] - 13s 229us/sample - loss: 0.7481 - accuracy: 0.7501 - val_loss: 0.5353 - val_accuracy: 0.8414
Epoch 9/20
55000/55000 [==============================] - 12s 225us/sample - loss: 0.7238 - accuracy: 0.7579 - val_loss: 0.5326 - val_accuracy: 0.8420
Epoch 10/20
55000/55000 [==============================] - 12s 222us/sample - loss: 0.6993 - accuracy: 0.7648 - val_loss: 0.5207 - val_accuracy: 0.8458
Epoch 11/20
55000/55000 [==============================] - 12s 222us/sample - loss: 0.6878 - accuracy: 0.7677 - val_loss: 0.5111 - val_accuracy: 0.8478
Epoch 12/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.6640 - accuracy: 0.7769 - val_loss: 0.5103 - val_accuracy: 0.8474
Epoch 13/20
55000/55000 [==============================] - 13s 243us/sample - loss: 0.6516 - accuracy: 0.7794 - val_loss: 0.5073 - val_accuracy: 0.8494
Epoch 14/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.6374 - accuracy: 0.7845 - val_loss: 0.5106 - val_accuracy: 0.8528
Epoch 15/20
55000/55000 [==============================] - 13s 234us/sample - loss: 0.6228 - accuracy: 0.7924 - val_loss: 0.5013 - val_accuracy: 0.8534
Epoch 16/20
55000/55000 [==============================] - 13s 231us/sample - loss: 0.6058 - accuracy: 0.7943 - val_loss: 0.4987 - val_accuracy: 0.8558
Epoch 17/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.5989 - accuracy: 0.7980 - val_loss: 0.4985 - val_accuracy: 0.8556
Epoch 18/20
55000/55000 [==============================] - 13s 232us/sample - loss: 0.5835 - accuracy: 0.8036 - val_loss: 0.4997 - val_accuracy: 0.8566
Epoch 19/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.5795 - accuracy: 0.8058 - val_loss: 0.4907 - val_accuracy: 0.8596
Epoch 20/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.5730 - accuracy: 0.8054 - val_loss: 0.4887 - val_accuracy: 0.8606
10000/1 [================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 1s 81us/sample - loss: 0.4401 - accuracy: 0.8478
[0.564703483581543, 0.8478]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
batch_normalization (BatchNo (None, 300)               1200      
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
batch_normalization_1 (Batch (None, 100)               400       
_________________________________________________________________
alpha_dropout (AlphaDropout) (None, 100)               0         
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 268,210
Trainable params: 267,410
Non-trainable params: 800
_________________________________________________________________
None

通過對dropout的使用，我們發現這個資料集的過擬合現象並不嚴重，它的識別準確率比之前反而有所降低。

wide_deep模型

google16年釋出，可用於分類和迴歸

稀疏特徵

離散值特徵，比如說一個人的性別資訊，只能從兩個值裡面去進行選擇。即類別型別的特徵。
可以使用One-hot表示。

比如：專業={計算機,人文,其他}。人文=[0, 1, 0]

比如: 詞表={人工智慧,你,他,網路,...}，他=[0, 0, 1, 0, ...]

叉乘={(計算機,人工智慧),(計算機,你),...}，這裡表示"專業"中的每個詞跟"詞表"中的每個詞進行組合。叉乘是可以用來刻畫一個樣本的。

比如說一個物體的所有特徵，我們都把它表示成離散特徵，我們就可以用叉乘來精確的刻畫一個樣本，叉乘可以精確的刻畫到所有樣本的所有可能性。

叉乘之後
- 稀疏特徵做叉乘獲取共現資訊
- 實現記憶的效果

如果一個物體可以用離散特徵去表達它的全部資訊，那麼這個物體的總量也是確定的，那麼叉乘之後的總的集合包含了所有物體的所有可能性。設想我們有一種非常大的模型，這個模型可以把所有樣本都記住。當我們遇到新的樣本的時候，那麼這個樣本肯定也是這個大集合中的一個。所以如果我們的模型能夠記住大集合中的每一個樣本，那麼對於任何新的樣本它都可以100%的去進行正確的預測。

稀疏特徵——優缺點

優點
- 有效，廣泛用於工業界
缺點
- 需要人工設計
- 可能過擬合，所有特徵都叉乘，相當於記住每一個樣本
- 泛化能力差，沒出現過就不會起效果

密集特徵

向量表達

比如：詞表={人工智慧,你,他,網路,...}，他=[0.3, 0.2, 0.6, (n維向量)]，我們可以用向量之間的距離來表示這兩個詞對應的距離。

Word2vec工具

可以參考Tensorflow深度學習演算法整理(二) 中的內容

密集特徵的優缺點

優點
- 帶有語義資訊，不同向量之間有相關性
- 相容沒有出現過的特徵組合
- 更少人工參與
缺點
- 過度泛化，推薦不怎麼相關的產品

Wide & Deep模型結構

Wide & Deep Vs Wide

在左邊的Wide模型中，我們可以看到它只有一層，輸入就是稀疏特徵，也就是一個One-hot表達。One-hot表達的每個分量都直接連線到輸出單元上去。在右邊的Wide & Deep模型，它的左半部分也就是一個Wide模型，它的右半部分是一個Deep模型，這個Deep模型就是有多層的神經網路，在這裡，我們將輸入的資料表現為一個密集的向量表達，在密集表達之上有一個多層的神經網路，在經過多層神經網路之後再連線到輸出的神經單元上去。

Wide & Deep Vs Deep

在這裡，右圖是一個Deep模型，它跟Wide & Deep模型的區別就在於它沒有連線稀疏特徵的那一塊。

這是google商店中一個推薦演算法的模型圖，它就是一個典型的Wide & Deep模型，它有很多的特徵，其中使用者已經安裝的軟體和展示的軟體做了叉乘，作為Wide模型的輸入。而其他的特徵，有些做了Embedding，有些沒有做Embedding，把它們拼接起來，作為一個總的特徵(密集表示)，再經過多層的神經網路，形成一個Deep的部分，然後再使用一個邏輯迴歸的損失函式把Wide和Deep給組合起來。現在我們先來看一個加利福尼亞的一個房價的迴歸問題

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split

if __name__ == "__main__":

    housing = fetch_california_housing()
    print(housing.DESCR)
    print(housing.data.shape)
    print(housing.target.shape)
    pprint.pprint(housing.data[0:5])
    pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    print(X_train.shape, y_train.shape)
    print(X_valid.shape, y_valid.shape)
    print(X_test.shape, y_test.shape)

執行結果

.. _california_housing_dataset:

California Housing dataset
--------------------------

**Data Set Characteristics:**

    :Number of Instances: 20640

    :Number of Attributes: 8 numeric, predictive attributes and the target

    :Attribute Information:
        - MedInc        median income in block
        - HouseAge      median house age in block
        - AveRooms      average number of rooms
        - AveBedrms     average number of bedrooms
        - Population    block population
        - AveOccup      average house occupancy
        - Latitude      house block latitude
        - Longitude     house block longitude

    :Missing Attribute Values: None

This dataset was obtained from the StatLib repository.
http://lib.stat.cmu.edu/datasets/

The target variable is the median house value for California districts.

This dataset was derived from the 1990 U.S. census, using one row per census
block group. A block group is the smallest geographical unit for which the U.S.
Census Bureau publishes sample data (a block group typically has a population
of 600 to 3,000 people).

It can be downloaded/loaded using the
:func:`sklearn.datasets.fetch_california_housing` function.

.. topic:: References

    - Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,
      Statistics and Probability Letters, 33 (1997) 291-297

(20640, 8)
(20640,)
array([[ 8.32520000e+00,  4.10000000e+01,  6.98412698e+00,
         1.02380952e+00,  3.22000000e+02,  2.55555556e+00,
         3.78800000e+01, -1.22230000e+02],
       [ 8.30140000e+00,  2.10000000e+01,  6.23813708e+00,
         9.71880492e-01,  2.40100000e+03,  2.10984183e+00,
         3.78600000e+01, -1.22220000e+02],
       [ 7.25740000e+00,  5.20000000e+01,  8.28813559e+00,
         1.07344633e+00,  4.96000000e+02,  2.80225989e+00,
         3.78500000e+01, -1.22240000e+02],
       [ 5.64310000e+00,  5.20000000e+01,  5.81735160e+00,
         1.07305936e+00,  5.58000000e+02,  2.54794521e+00,
         3.78500000e+01, -1.22250000e+02],
       [ 3.84620000e+00,  5.20000000e+01,  6.28185328e+00,
         1.08108108e+00,  5.65000000e+02,  2.18146718e+00,
         3.78500000e+01, -1.22250000e+02]])
array([4.526, 3.585, 3.521, 3.413, 3.422])
(11610, 8) (11610,)
(3870, 8) (3870,)
(5160, 8) (5160,)

我們可以看到這裡有20640條樣本資料，維度為8，說明影響房價的有8個特徵。其中，我們列印了5條樣本資料和標籤資料。並將其拆分為訓練資料集，驗證資料集和測試資料集。現在我們來進行模型訓練和列印學習曲線

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    model = keras.models.Sequential([
        layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
        layers.Dense(1)
    ])
    print(model.summary())
    # 使用均方誤差損失函式和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.01))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, 30)                270       
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 31        
=================================================================
Total params: 301
Trainable params: 301
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 88us/sample - loss: 1.3027 - val_loss: 2.2434
Epoch 2/100
11610/11610 [==============================] - 1s 45us/sample - loss: 4.6891 - val_loss: 0.4434
Epoch 3/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.4092 - val_loss: 0.3887
Epoch 4/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3811 - val_loss: 0.3797
Epoch 5/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3718 - val_loss: 0.3839
Epoch 6/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3652 - val_loss: 0.3714
Epoch 7/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3620 - val_loss: 0.3662
Epoch 8/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3587 - val_loss: 0.3715
Epoch 9/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3555 - val_loss: 0.3632
Epoch 10/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3525 - val_loss: 0.3620
Epoch 11/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3518 - val_loss: 0.3566
Epoch 12/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3486 - val_loss: 0.3544
Epoch 13/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3466 - val_loss: 0.3494
Epoch 14/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3468 - val_loss: 0.3458
Epoch 15/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3417 - val_loss: 0.3487
Epoch 16/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3412 - val_loss: 0.3449
Epoch 17/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3386 - val_loss: 0.3592
Epoch 18/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3372 - val_loss: 0.3412
Epoch 19/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3364 - val_loss: 0.3426
Epoch 20/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3336 - val_loss: 0.3462
Epoch 21/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3317 - val_loss: 0.3388
Epoch 22/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3300 - val_loss: 0.3461
Epoch 23/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3292 - val_loss: 0.3355
Epoch 24/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3281 - val_loss: 0.3344
Epoch 25/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3266 - val_loss: 0.3374
Epoch 26/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3283 - val_loss: 0.3348
Epoch 27/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3242 - val_loss: 0.3395
Epoch 28/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3247 - val_loss: 0.3322
Epoch 29/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3233 - val_loss: 0.3288
Epoch 30/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3216 - val_loss: 0.3353
Epoch 31/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3216 - val_loss: 0.3293
Epoch 32/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3214 - val_loss: 0.3312
Epoch 33/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3201 - val_loss: 0.3290
Epoch 34/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3189 - val_loss: 0.3319
5160/1 [==================================================] - 0s 19us/sample - loss: 0.4063
0.342487285580746

這裡我們發現，我們設定了100次的迭代，但是隻到了34次並沒有到100次就停止了，再根據學習曲線的情況，我們發現是EarlyStopping的回撥函式起到了作用，是梯度下降已經到了瓶頸，趨於平緩。而測試資料集的損失函式值為0.342。

現在我們來將這個簡單的迴歸模型改成Wide & Deep模型

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函式式API Deep模型
    input = layers.Input(shape=X_train.shape[1:])
    hidden1 = layers.Dense(30, activation='relu')(input)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # input是Wide模型的輸入，hidden2是Deep模型的輸出
    concat = layers.concatenate([input, hidden2])
    output = layers.Dense(1)(concat)
    model = models.Model(inputs=[input], outputs=[output])
    print(model.summary())
    # 使用均方誤差損失函式和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 8)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           270         input_1[0][0]                    
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 38)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            39          concatenate[0][0]                
==================================================================================================
Total params: 1,239
Trainable params: 1,239
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 107us/sample - loss: 1.7756 - val_loss: 0.8341
Epoch 2/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.7057 - val_loss: 0.7208
Epoch 3/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.6495 - val_loss: 0.6740
Epoch 4/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.6138 - val_loss: 0.6451
Epoch 5/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5884 - val_loss: 0.6189
Epoch 6/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.5682 - val_loss: 0.6013
Epoch 7/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5515 - val_loss: 0.5818
Epoch 8/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.5387 - val_loss: 0.5675
Epoch 9/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5272 - val_loss: 0.5547
Epoch 10/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5180 - val_loss: 0.5441
Epoch 11/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5085 - val_loss: 0.5336
Epoch 12/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5012 - val_loss: 0.5285
Epoch 13/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4931 - val_loss: 0.5167
Epoch 14/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4881 - val_loss: 0.5128
Epoch 15/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4813 - val_loss: 0.5043
Epoch 16/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4760 - val_loss: 0.4983
Epoch 17/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4710 - val_loss: 0.4928
Epoch 18/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4655 - val_loss: 0.4917
Epoch 19/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4621 - val_loss: 0.4818
Epoch 20/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4571 - val_loss: 0.4776
Epoch 21/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4531 - val_loss: 0.4779
Epoch 22/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4501 - val_loss: 0.4696
Epoch 23/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4455 - val_loss: 0.4650
Epoch 24/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4425 - val_loss: 0.4612
Epoch 25/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4404 - val_loss: 0.4592
Epoch 26/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4367 - val_loss: 0.4542
Epoch 27/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4336 - val_loss: 0.4507
Epoch 28/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4310 - val_loss: 0.4492
Epoch 29/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4278 - val_loss: 0.4470
Epoch 30/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4257 - val_loss: 0.4429
Epoch 31/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4227 - val_loss: 0.4398
Epoch 32/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4208 - val_loss: 0.4375
Epoch 33/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4183 - val_loss: 0.4346
Epoch 34/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4162 - val_loss: 0.4327
Epoch 35/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4136 - val_loss: 0.4306
Epoch 36/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4116 - val_loss: 0.4275
Epoch 37/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4096 - val_loss: 0.4258
Epoch 38/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4075 - val_loss: 0.4236
Epoch 39/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4053 - val_loss: 0.4213
Epoch 40/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4037 - val_loss: 0.4191
Epoch 41/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4016 - val_loss: 0.4180
Epoch 42/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4003 - val_loss: 0.4157
Epoch 43/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3981 - val_loss: 0.4129
Epoch 44/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3962 - val_loss: 0.4130
Epoch 45/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3949 - val_loss: 0.4094
Epoch 46/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3929 - val_loss: 0.4080
Epoch 47/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3915 - val_loss: 0.4067
Epoch 48/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3898 - val_loss: 0.4044
Epoch 49/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3885 - val_loss: 0.4032
Epoch 50/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3868 - val_loss: 0.4019
Epoch 51/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3850 - val_loss: 0.3995
Epoch 52/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3839 - val_loss: 0.3993
Epoch 53/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3825 - val_loss: 0.3971
Epoch 54/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3809 - val_loss: 0.3951
Epoch 55/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3798 - val_loss: 0.3940
Epoch 56/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3788 - val_loss: 0.3929
Epoch 57/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3775 - val_loss: 0.3916
Epoch 58/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3760 - val_loss: 0.3906
Epoch 59/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3749 - val_loss: 0.3900
Epoch 60/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3742 - val_loss: 0.3880
Epoch 61/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3726 - val_loss: 0.3866
Epoch 62/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3718 - val_loss: 0.3863
Epoch 63/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3706 - val_loss: 0.3853
Epoch 64/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.3698 - val_loss: 0.3834
Epoch 65/100
11610/11610 [==============================] - 1s 71us/sample - loss: 0.3687 - val_loss: 0.3834
Epoch 66/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3679 - val_loss: 0.3818
Epoch 67/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3668 - val_loss: 0.3811
Epoch 68/100
11610/11610 [==============================] - 1s 78us/sample - loss: 0.3658 - val_loss: 0.3801
Epoch 69/100
11610/11610 [==============================] - 1s 76us/sample - loss: 0.3649 - val_loss: 0.3787
Epoch 70/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.3639 - val_loss: 0.3781
Epoch 71/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3634 - val_loss: 0.3769
Epoch 72/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3621 - val_loss: 0.3762
Epoch 73/100
11610/11610 [==============================] - 1s 66us/sample - loss: 0.3615 - val_loss: 0.3762
Epoch 74/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3606 - val_loss: 0.3746
Epoch 75/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3596 - val_loss: 0.3749
Epoch 76/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3590 - val_loss: 0.3736
Epoch 77/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3580 - val_loss: 0.3740
Epoch 78/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3576 - val_loss: 0.3721
Epoch 79/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3562 - val_loss: 0.3712
Epoch 80/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3558 - val_loss: 0.3710
Epoch 81/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3549 - val_loss: 0.3710
Epoch 82/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3541 - val_loss: 0.3707
Epoch 83/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3537 - val_loss: 0.3690
Epoch 84/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3530 - val_loss: 0.3691
Epoch 85/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3520 - val_loss: 0.3679
Epoch 86/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3514 - val_loss: 0.3673
Epoch 87/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3508 - val_loss: 0.3671
Epoch 88/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3500 - val_loss: 0.3662
Epoch 89/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3496 - val_loss: 0.3655
Epoch 90/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3487 - val_loss: 0.3661
Epoch 91/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3484 - val_loss: 0.3658
Epoch 92/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3479 - val_loss: 0.3641
Epoch 93/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3468 - val_loss: 0.3637
Epoch 94/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3462 - val_loss: 0.3640
Epoch 95/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3456 - val_loss: 0.3629
Epoch 96/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3449 - val_loss: 0.3621
Epoch 97/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3444 - val_loss: 0.3621
Epoch 98/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3431 - val_loss: 0.3622
Epoch 99/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3434 - val_loss: 0.3606
Epoch 100/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3430 - val_loss: 0.3604
5160/1 [========================================================] - 0s 24us/sample - loss: 0.4557
0.36289026386053985

在這裡，我們使用的是函數語言程式設計來實現的Wide & Deep模型，下面我們來使用子類API來實現這個模型

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函式式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的輸入，hidden2是Deep模型的輸出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子類API，父類為keras.models.Model
    class WideDeepModel(models.Model):

        def __init__(self):
            super(WideDeepModel, self).__init__()
            # 定義模型的層次
            self.hidden1_layer = layers.Dense(30, activation='relu')
            self.hidden2_layer = layers.Dense(30, activation='relu')
            self.output_layer = layers.Dense(1)


        def call(self, input):
            # 完成模型的正向計算
            hidden1 = self.hidden1_layer(input)
            hidden2 = self.hidden2_layer(hidden1)
            concat = layers.concatenate([input, hidden2])
            output = self.output_layer(concat)
            return output

    model = WideDeepModel()
    # 也可以使用層堆疊
    # model = models.Sequential([
    #     WideDeepModel()
    # ])
    model.build(input_shape=(None, 8))
    print(model.summary())
    # 使用均方誤差損失函式和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Model: "wide_deep_model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                multiple                  270       
_________________________________________________________________
dense_1 (Dense)              multiple                  930       
_________________________________________________________________
dense_2 (Dense)              multiple                  39        
=================================================================
Total params: 1,239
Trainable params: 1,239
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 107us/sample - loss: 2.0999 - val_loss: 0.8217
Epoch 2/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.6996 - val_loss: 0.7236
Epoch 3/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.6376 - val_loss: 0.6704
Epoch 4/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.6003 - val_loss: 0.6322
Epoch 5/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5720 - val_loss: 0.6104
Epoch 6/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5499 - val_loss: 0.5869
Epoch 7/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5324 - val_loss: 0.5653
Epoch 8/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.5188 - val_loss: 0.5499
Epoch 9/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5075 - val_loss: 0.5401
Epoch 10/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4970 - val_loss: 0.5268
Epoch 11/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4902 - val_loss: 0.5204
Epoch 12/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.4819 - val_loss: 0.5103
Epoch 13/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.4759 - val_loss: 0.5056
Epoch 14/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.4697 - val_loss: 0.4977
Epoch 15/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.4654 - val_loss: 0.4931
Epoch 16/100
11610/11610 [==============================] - 1s 83us/sample - loss: 0.4597 - val_loss: 0.4928
Epoch 17/100
11610/11610 [==============================] - 1s 78us/sample - loss: 0.4538 - val_loss: 0.4821
Epoch 18/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.4506 - val_loss: 0.4765
Epoch 19/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.4465 - val_loss: 0.4744
Epoch 20/100
11610/11610 [==============================] - 1s 58us/sample - loss: 0.4430 - val_loss: 0.4762
Epoch 21/100
11610/11610 [==============================] - 1s 99us/sample - loss: 0.4391 - val_loss: 0.4668
Epoch 22/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.4361 - val_loss: 0.4641
Epoch 23/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.4328 - val_loss: 0.4587
Epoch 24/100
11610/11610 [==============================] - 1s 66us/sample - loss: 0.4293 - val_loss: 0.4565
Epoch 25/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4266 - val_loss: 0.4525
Epoch 26/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4237 - val_loss: 0.4481
Epoch 27/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4210 - val_loss: 0.4452
Epoch 28/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4178 - val_loss: 0.4426
Epoch 29/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.4163 - val_loss: 0.4410
Epoch 30/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.4135 - val_loss: 0.4372
Epoch 31/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.4111 - val_loss: 0.4350
Epoch 32/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4096 - val_loss: 0.4329
Epoch 33/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4075 - val_loss: 0.4298
Epoch 34/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4053 - val_loss: 0.4293
Epoch 35/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4035 - val_loss: 0.4263
Epoch 36/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4018 - val_loss: 0.4236
Epoch 37/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3997 - val_loss: 0.4221
Epoch 38/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3977 - val_loss: 0.4207
Epoch 39/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3962 - val_loss: 0.4175
Epoch 40/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3947 - val_loss: 0.4152
Epoch 41/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3927 - val_loss: 0.4140
Epoch 42/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3914 - val_loss: 0.4131
Epoch 43/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3897 - val_loss: 0.4102
Epoch 44/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3883 - val_loss: 0.4088
Epoch 45/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3869 - val_loss: 0.4063
Epoch 46/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3858 - val_loss: 0.4053
Epoch 47/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3841 - val_loss: 0.4041
Epoch 48/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3832 - val_loss: 0.4029
Epoch 49/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3818 - val_loss: 0.4002
Epoch 50/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.3807 - val_loss: 0.3987
Epoch 51/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3791 - val_loss: 0.3970
Epoch 52/100
11610/11610 [==============================] - 1s 73us/sample - loss: 0.3779 - val_loss: 0.3970
Epoch 53/100
11610/11610 [==============================] - 1s 98us/sample - loss: 0.3768 - val_loss: 0.3964
Epoch 54/100
11610/11610 [==============================] - 1s 92us/sample - loss: 0.3756 - val_loss: 0.3941
Epoch 55/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3746 - val_loss: 0.3926
Epoch 56/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3739 - val_loss: 0.3914
Epoch 57/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3727 - val_loss: 0.3896
Epoch 58/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3718 - val_loss: 0.3879
Epoch 59/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3701 - val_loss: 0.3900
Epoch 60/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3696 - val_loss: 0.3865
Epoch 61/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3683 - val_loss: 0.3855
Epoch 62/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3674 - val_loss: 0.3860
Epoch 63/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3669 - val_loss: 0.3831
Epoch 64/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3652 - val_loss: 0.3820
Epoch 65/100
11610/11610 [==============================] - 1s 76us/sample - loss: 0.3645 - val_loss: 0.3818
Epoch 66/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.3641 - val_loss: 0.3794
Epoch 67/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.3623 - val_loss: 0.3799
Epoch 68/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.3623 - val_loss: 0.3782
Epoch 69/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3606 - val_loss: 0.3766
Epoch 70/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.3599 - val_loss: 0.3766
Epoch 71/100
11610/11610 [==============================] - 1s 87us/sample - loss: 0.3589 - val_loss: 0.3748
Epoch 72/100
11610/11610 [==============================] - 1s 100us/sample - loss: 0.3582 - val_loss: 0.3743
Epoch 73/100
11610/11610 [==============================] - 1s 60us/sample - loss: 0.3573 - val_loss: 0.3728
Epoch 74/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3564 - val_loss: 0.3735
Epoch 75/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.3562 - val_loss: 0.3718
Epoch 76/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.3557 - val_loss: 0.3714
Epoch 77/100
11610/11610 [==============================] - 1s 85us/sample - loss: 0.3548 - val_loss: 0.3703
Epoch 78/100
11610/11610 [==============================] - 1s 93us/sample - loss: 0.3540 - val_loss: 0.3700
Epoch 79/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3528 - val_loss: 0.3685
Epoch 80/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3526 - val_loss: 0.3678
Epoch 81/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3512 - val_loss: 0.3678
Epoch 82/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3506 - val_loss: 0.3672
Epoch 83/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3508 - val_loss: 0.3657
Epoch 84/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3495 - val_loss: 0.3669
Epoch 85/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3491 - val_loss: 0.3670
Epoch 86/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3490 - val_loss: 0.3642
Epoch 87/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3486 - val_loss: 0.3642
Epoch 88/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3478 - val_loss: 0.3643
Epoch 89/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3474 - val_loss: 0.3631
Epoch 90/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3467 - val_loss: 0.3614
Epoch 91/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3451 - val_loss: 0.3604
Epoch 92/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3444 - val_loss: 0.3606
Epoch 93/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3436 - val_loss: 0.3592
Epoch 94/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3435 - val_loss: 0.3588
Epoch 95/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3585
Epoch 96/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3576
Epoch 97/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3578
Epoch 98/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3419 - val_loss: 0.3569
Epoch 99/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3412 - val_loss: 0.3572
Epoch 100/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3404 - val_loss: 0.3551
5160/1 [===========================================================] - 0s 45us/sample - loss: 0.4603
0.36134041002554484

由於Wide & Deep模型可以分別挑選特徵來進行輸入，現在我們來改成多輸入的模式

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函式式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的輸入，hidden2是Deep模型的輸出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子類API，父類為keras.models.Model
    # class WideDeepModel(models.Model):
    #
    #     def __init__(self):
    #         super(WideDeepModel, self).__init__()
    #         # 定義模型的層次
    #         self.hidden1_layer = layers.Dense(30, activation='relu')
    #         self.hidden2_layer = layers.Dense(30, activation='relu')
    #         self.output_layer = layers.Dense(1)
    #
    #
    #     def call(self, input):
    #         # 完成模型的正向計算
    #         hidden1 = self.hidden1_layer(input)
    #         hidden2 = self.hidden2_layer(hidden1)
    #         concat = layers.concatenate([input, hidden2])
    #         output = self.output_layer(concat)
    #         return output
    #
    # model = WideDeepModel()
    # # 也可以使用層堆疊
    # # model = models.Sequential([
    # #     WideDeepModel()
    # # ])
    # model.build(input_shape=(None, 8))
    # 多輸入 wide模型輸入前5個特徵，deep模型輸入後6個特徵
    input_wide = layers.Input(shape=[5])
    input_deep = layers.Input(shape=[6])
    hidden1 = layers.Dense(30, activation='relu')(input_deep)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    concat = layers.concatenate([input_wide, hidden2])
    output = layers.Dense(1)(concat)
    model = models.Model(inputs=[input_wide, input_deep], outputs=[output])
    print(model.summary())
    # 使用均方誤差損失函式和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    X_train_wide = X_train[:, :5]
    X_train_deep = X_train[:, 2:]
    X_valid_wide = X_valid[:, :5]
    X_valid_deep = X_valid[:, 2:]
    X_test_wide = X_test[:, :5]
    X_test_deep = X_test[:, 2:]
    history = model.fit([X_train_wide, X_train_deep], y_train,
                        validation_data=([X_valid_wide, X_valid_deep], y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate([X_test_wide, X_test_deep], y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 6)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           210         input_2[0][0]                    
__________________________________________________________________________________________________
input_1 (InputLayer)            [(None, 5)]          0                                            
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 35)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            36          concatenate[0][0]                
==================================================================================================
Total params: 1,176
Trainable params: 1,176
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 2s 138us/sample - loss: 1.9978 - val_loss: 0.9837
Epoch 2/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.8110 - val_loss: 0.7862
Epoch 3/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.7007 - val_loss: 0.7168
Epoch 4/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.6489 - val_loss: 0.6742
Epoch 5/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.6118 - val_loss: 0.6378
Epoch 6/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5831 - val_loss: 0.6127
Epoch 7/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5607 - val_loss: 0.5910
Epoch 8/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5427 - val_loss: 0.5736
Epoch 9/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.5275 - val_loss: 0.5589
Epoch 10/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5144 - val_loss: 0.5455
Epoch 11/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5033 - val_loss: 0.5353
Epoch 12/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4939 - val_loss: 0.5258
Epoch 13/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4853 - val_loss: 0.5156
Epoch 14/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4784 - val_loss: 0.5084
Epoch 15/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4718 - val_loss: 0.5035
Epoch 16/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4657 - val_loss: 0.4952
Epoch 17/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4606 - val_loss: 0.4887
Epoch 18/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4560 - val_loss: 0.4840
Epoch 19/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4517 - val_loss: 0.4805
Epoch 20/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4477 - val_loss: 0.4773
Epoch 21/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4451 - val_loss: 0.4727
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4423 - val_loss: 0.4697
Epoch 23/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4397 - val_loss: 0.4708
Epoch 24/100
11610/11610 [==============================] - 1s 72us/sample - loss: 0.4375 - val_loss: 0.4650
Epoch 25/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4350 - val_loss: 0.4633
Epoch 26/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4329 - val_loss: 0.4593
Epoch 27/100
11610/11610 [==============================] - 1s 88us/sample - loss: 0.4310 - val_loss: 0.4569
Epoch 28/100
11610/11610 [==============================] - 1s 99us/sample - loss: 0.4294 - val_loss: 0.4558
Epoch 29/100
11610/11610 [==============================] - 1s 97us/sample - loss: 0.4276 - val_loss: 0.4521
Epoch 30/100
11610/11610 [==============================] - 1s 120us/sample - loss: 0.4254 - val_loss: 0.4494
Epoch 31/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4239 - val_loss: 0.4484
Epoch 32/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.4219 - val_loss: 0.4470
Epoch 33/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4209 - val_loss: 0.4456
Epoch 34/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4194 - val_loss: 0.4433
Epoch 35/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4179 - val_loss: 0.4427
Epoch 36/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4163 - val_loss: 0.4399
Epoch 37/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4150 - val_loss: 0.4399
Epoch 38/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4137 - val_loss: 0.4370
Epoch 39/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4122 - val_loss: 0.4364
Epoch 40/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4112 - val_loss: 0.4369
Epoch 41/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4098 - val_loss: 0.4329
Epoch 42/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4088 - val_loss: 0.4326
Epoch 43/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4077 - val_loss: 0.4299
Epoch 44/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4066 - val_loss: 0.4304
Epoch 45/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4054 - val_loss: 0.4284
Epoch 46/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.4043 - val_loss: 0.4286
Epoch 47/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.4036 - val_loss: 0.4263
Epoch 48/100
11610/11610 [==============================] - 1s 88us/sample - loss: 0.4026 - val_loss: 0.4251
Epoch 49/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.4014 - val_loss: 0.4240
Epoch 50/100
11610/11610 [==============================] - 1s 96us/sample - loss: 0.4007 - val_loss: 0.4231
Epoch 51/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.3998 - val_loss: 0.4216
Epoch 52/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3990 - val_loss: 0.4235
Epoch 53/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3981 - val_loss: 0.4211
Epoch 54/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3976 - val_loss: 0.4210
Epoch 55/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3965 - val_loss: 0.4217
Epoch 56/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3959 - val_loss: 0.4174
Epoch 57/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3953 - val_loss: 0.4175
Epoch 58/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.3943 - val_loss: 0.4163
Epoch 59/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3935 - val_loss: 0.4156
Epoch 60/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3928 - val_loss: 0.4150
Epoch 61/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3922 - val_loss: 0.4147
Epoch 62/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3913 - val_loss: 0.4137
Epoch 63/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3906 - val_loss: 0.4128
Epoch 64/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3898 - val_loss: 0.4113
Epoch 65/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3894 - val_loss: 0.4121
Epoch 66/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3884 - val_loss: 0.4110
Epoch 67/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3878 - val_loss: 0.4129
Epoch 68/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3873 - val_loss: 0.4109
Epoch 69/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3864 - val_loss: 0.4101
Epoch 70/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3858 - val_loss: 0.4092
Epoch 71/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3853 - val_loss: 0.4073
Epoch 72/100
11610/11610 [==============================] - 1s 94us/sample - loss: 0.3847 - val_loss: 0.4076
Epoch 73/100
11610/11610 [==============================] - 1s 98us/sample - loss: 0.3842 - val_loss: 0.4059
Epoch 74/100
11610/11610 [==============================] - 1s 80us/sample - loss: 0.3834 - val_loss: 0.4047
Epoch 75/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3826 - val_loss: 0.4056
Epoch 76/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.3822 - val_loss: 0.4031
Epoch 77/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3814 - val_loss: 0.4049
Epoch 78/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3812 - val_loss: 0.4029
Epoch 79/100
11610/11610 [==============================] - 1s 59us/sample - loss: 0.3800 - val_loss: 0.4022
Epoch 80/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3794 - val_loss: 0.4004
Epoch 81/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3793 - val_loss: 0.4004
Epoch 82/100
11610/11610 [==============================] - 1s 79us/sample - loss: 0.3784 - val_loss: 0.3998
Epoch 83/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3778 - val_loss: 0.3988
Epoch 84/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3770 - val_loss: 0.3981
Epoch 85/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3764 - val_loss: 0.3987
Epoch 86/100
11610/11610 [==============================] - 1s 83us/sample - loss: 0.3756 - val_loss: 0.3970
Epoch 87/100
11610/11610 [==============================] - 1s 89us/sample - loss: 0.3751 - val_loss: 0.3969
Epoch 88/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3747 - val_loss: 0.3950
Epoch 89/100
11610/11610 [==============================] - 1s 90us/sample - loss: 0.3740 - val_loss: 0.3951
Epoch 90/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.3733 - val_loss: 0.3945
Epoch 91/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3727 - val_loss: 0.3937
Epoch 92/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3721 - val_loss: 0.3937
Epoch 93/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3717 - val_loss: 0.3932
Epoch 94/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3709 - val_loss: 0.3930
Epoch 95/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3703 - val_loss: 0.3919
Epoch 96/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3695 - val_loss: 0.3921
Epoch 97/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3692 - val_loss: 0.3909
Epoch 98/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3687 - val_loss: 0.3892
Epoch 99/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3676 - val_loss: 0.3902
Epoch 100/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3677 - val_loss: 0.3888
5160/1 [=======================================================] - 0s 24us/sample - loss: 0.4975
0.3788368522658829

有了多輸入就會有多輸出

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函式式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的輸入，hidden2是Deep模型的輸出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子類API，父類為keras.models.Model
    # class WideDeepModel(models.Model):
    #
    #     def __init__(self):
    #         super(WideDeepModel, self).__init__()
    #         # 定義模型的層次
    #         self.hidden1_layer = layers.Dense(30, activation='relu')
    #         self.hidden2_layer = layers.Dense(30, activation='relu')
    #         self.output_layer = layers.Dense(1)
    #
    #
    #     def call(self, input):
    #         # 完成模型的正向計算
    #         hidden1 = self.hidden1_layer(input)
    #         hidden2 = self.hidden2_layer(hidden1)
    #         concat = layers.concatenate([input, hidden2])
    #         output = self.output_layer(concat)
    #         return output
    #
    # model = WideDeepModel()
    # # 也可以使用層堆疊
    # # model = models.Sequential([
    # #     WideDeepModel()
    # # ])
    # model.build(input_shape=(None, 8))
    # 多輸入 wide模型輸入前5個特徵，deep模型輸入後6個特徵
    input_wide = layers.Input(shape=[5])
    input_deep = layers.Input(shape=[6])
    hidden1 = layers.Dense(30, activation='relu')(input_deep)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    concat = layers.concatenate([input_wide, hidden2])
    # 多輸出
    output1 = layers.Dense(1)(concat)
    output2 = layers.Dense(1)(hidden2)
    model = models.Model(inputs=[input_wide, input_deep], outputs=[output1, output2])
    print(model.summary())
    # 使用均方誤差損失函式和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    X_train_wide = X_train[:, :5]
    X_train_deep = X_train[:, 2:]
    X_valid_wide = X_valid[:, :5]
    X_valid_deep = X_valid[:, 2:]
    X_test_wide = X_test[:, :5]
    X_test_deep = X_test[:, 2:]
    history = model.fit([X_train_wide, X_train_deep], [y_train, y_train],
                        validation_data=([X_valid_wide, X_valid_deep], [y_valid, y_valid]),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate([X_test_wide, X_test_deep], [y_test, y_test]))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 6)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           210         input_2[0][0]                    
__________________________________________________________________________________________________
input_1 (InputLayer)            [(None, 5)]          0                                            
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 35)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            36          concatenate[0][0]                
__________________________________________________________________________________________________
dense_3 (Dense)                 (None, 1)            31          dense_1[0][0]                    
==================================================================================================
Total params: 1,207
Trainable params: 1,207
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 2s 151us/sample - loss: 4.6904 - dense_2_loss: 2.2887 - dense_3_loss: 2.4007 - val_loss: 2.2365 - val_dense_2_loss: 0.8763 - val_dense_3_loss: 1.3602
Epoch 2/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.8804 - dense_2_loss: 0.7132 - dense_3_loss: 1.1668 - val_loss: 1.8578 - val_dense_2_loss: 0.6893 - val_dense_3_loss: 1.1685
Epoch 3/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.6375 - dense_2_loss: 0.6051 - dense_3_loss: 1.0320 - val_loss: 1.6503 - val_dense_2_loss: 0.6125 - val_dense_3_loss: 1.0377
Epoch 4/100
11610/11610 [==============================] - 1s 63us/sample - loss: 1.4545 - dense_2_loss: 0.5502 - dense_3_loss: 0.9039 - val_loss: 1.4679 - val_dense_2_loss: 0.5670 - val_dense_3_loss: 0.9009
Epoch 5/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.3185 - dense_2_loss: 0.5166 - dense_3_loss: 0.8021 - val_loss: 1.3543 - val_dense_2_loss: 0.5365 - val_dense_3_loss: 0.8177
Epoch 6/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.2300 - dense_2_loss: 0.4951 - dense_3_loss: 0.7349 - val_loss: 1.2700 - val_dense_2_loss: 0.5162 - val_dense_3_loss: 0.7536
Epoch 7/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.1740 - dense_2_loss: 0.4805 - dense_3_loss: 0.6936 - val_loss: 1.2194 - val_dense_2_loss: 0.5032 - val_dense_3_loss: 0.7161
Epoch 8/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.1349 - dense_2_loss: 0.4695 - dense_3_loss: 0.6651 - val_loss: 1.1812 - val_dense_2_loss: 0.4888 - val_dense_3_loss: 0.6923
Epoch 9/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.1066 - dense_2_loss: 0.4612 - dense_3_loss: 0.6452 - val_loss: 1.1580 - val_dense_2_loss: 0.4876 - val_dense_3_loss: 0.6703
Epoch 10/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0847 - dense_2_loss: 0.4549 - dense_3_loss: 0.6300 - val_loss: 1.1370 - val_dense_2_loss: 0.4784 - val_dense_3_loss: 0.6585
Epoch 11/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0667 - dense_2_loss: 0.4483 - dense_3_loss: 0.6183 - val_loss: 1.1159 - val_dense_2_loss: 0.4717 - val_dense_3_loss: 0.6441
Epoch 12/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0524 - dense_2_loss: 0.4440 - dense_3_loss: 0.6086 - val_loss: 1.1014 - val_dense_2_loss: 0.4640 - val_dense_3_loss: 0.6373
Epoch 13/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0388 - dense_2_loss: 0.4397 - dense_3_loss: 0.5995 - val_loss: 1.0919 - val_dense_2_loss: 0.4677 - val_dense_3_loss: 0.6241
Epoch 14/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0277 - dense_2_loss: 0.4357 - dense_3_loss: 0.5921 - val_loss: 1.0758 - val_dense_2_loss: 0.4573 - val_dense_3_loss: 0.6184
Epoch 15/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.0185 - dense_2_loss: 0.4323 - dense_3_loss: 0.5861 - val_loss: 1.0668 - val_dense_2_loss: 0.4580 - val_dense_3_loss: 0.6087
Epoch 16/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0102 - dense_2_loss: 0.4295 - dense_3_loss: 0.5807 - val_loss: 1.0546 - val_dense_2_loss: 0.4518 - val_dense_3_loss: 0.6027
Epoch 17/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0019 - dense_2_loss: 0.4267 - dense_3_loss: 0.5751 - val_loss: 1.0487 - val_dense_2_loss: 0.4512 - val_dense_3_loss: 0.5974
Epoch 18/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.9950 - dense_2_loss: 0.4239 - dense_3_loss: 0.5714 - val_loss: 1.0379 - val_dense_2_loss: 0.4465 - val_dense_3_loss: 0.5913
Epoch 19/100
11610/11610 [==============================] - 1s 109us/sample - loss: 0.9884 - dense_2_loss: 0.4213 - dense_3_loss: 0.5668 - val_loss: 1.0312 - val_dense_2_loss: 0.4447 - val_dense_3_loss: 0.5864
Epoch 20/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.9820 - dense_2_loss: 0.4188 - dense_3_loss: 0.5630 - val_loss: 1.0274 - val_dense_2_loss: 0.4428 - val_dense_3_loss: 0.5844
Epoch 21/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.9757 - dense_2_loss: 0.4165 - dense_3_loss: 0.5590 - val_loss: 1.0197 - val_dense_2_loss: 0.4413 - val_dense_3_loss: 0.5783
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.9690 - dense_2_loss: 0.4140 - dense_3_loss: 0.5547 - val_loss: 1.0125 - val_dense_2_loss: 0.4372 - val_dense_3_loss: 0.5752
Epoch 23/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9633 - dense_2_loss: 0.4119 - dense_3_loss: 0.5514 - val_loss: 1.0073 - val_dense_2_loss: 0.4348 - val_dense_3_loss: 0.5723
Epoch 24/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9581 - dense_2_loss: 0.4105 - dense_3_loss: 0.5483 - val_loss: 1.0053 - val_dense_2_loss: 0.4386 - val_dense_3_loss: 0.5666
Epoch 25/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9543 - dense_2_loss: 0.4085 - dense_3_loss: 0.5457 - val_loss: 0.9971 - val_dense_2_loss: 0.4320 - val_dense_3_loss: 0.5650
Epoch 26/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9473 - dense_2_loss: 0.4054 - dense_3_loss: 0.5417 - val_loss: 0.9948 - val_dense_2_loss: 0.4335 - val_dense_3_loss: 0.5612
Epoch 27/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9427 - dense_2_loss: 0.4044 - dense_3_loss: 0.5383 - val_loss: 0.9884 - val_dense_2_loss: 0.4279 - val_dense_3_loss: 0.5604
Epoch 28/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9392 - dense_2_loss: 0.4028 - dense_3_loss: 0.5362 - val_loss: 0.9836 - val_dense_2_loss: 0.4280 - val_dense_3_loss: 0.5555
Epoch 29/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9333 - dense_2_loss: 0.4003 - dense_3_loss: 0.5329 - val_loss: 0.9803 - val_dense_2_loss: 0.4257 - val_dense_3_loss: 0.5544
Epoch 30/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9297 - dense_2_loss: 0.3990 - dense_3_loss: 0.5304 - val_loss: 0.9735 - val_dense_2_loss: 0.4229 - val_dense_3_loss: 0.5505
Epoch 31/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.9257 - dense_2_loss: 0.3978 - dense_3_loss: 0.5278 - val_loss: 0.9735 - val_dense_2_loss: 0.4216 - val_dense_3_loss: 0.5518
Epoch 32/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9202 - dense_2_loss: 0.3956 - dense_3_loss: 0.5247 - val_loss: 0.9665 - val_dense_2_loss: 0.4209 - val_dense_3_loss: 0.5455
Epoch 33/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9178 - dense_2_loss: 0.3946 - dense_3_loss: 0.5229 - val_loss: 0.9594 - val_dense_2_loss: 0.4180 - val_dense_3_loss: 0.5413
Epoch 34/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9121 - dense_2_loss: 0.3925 - dense_3_loss: 0.5197 - val_loss: 0.9559 - val_dense_2_loss: 0.4147 - val_dense_3_loss: 0.5411
Epoch 35/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9080 - dense_2_loss: 0.3910 - dense_3_loss: 0.5168 - val_loss: 0.9536 - val_dense_2_loss: 0.4119 - val_dense_3_loss: 0.5415
Epoch 36/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9046 - dense_2_loss: 0.3900 - dense_3_loss: 0.5146 - val_loss: 0.9511 - val_dense_2_loss: 0.4139 - val_dense_3_loss: 0.5371
Epoch 37/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9000 - dense_2_loss: 0.3882 - dense_3_loss: 0.5118 - val_loss: 0.9507 - val_dense_2_loss: 0.4199 - val_dense_3_loss: 0.5307
Epoch 38/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8990 - dense_2_loss: 0.3884 - dense_3_loss: 0.5110 - val_loss: 0.9416 - val_dense_2_loss: 0.4145 - val_dense_3_loss: 0.5270
Epoch 39/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8928 - dense_2_loss: 0.3859 - dense_3_loss: 0.5071 - val_loss: 0.9357 - val_dense_2_loss: 0.4092 - val_dense_3_loss: 0.5264
Epoch 40/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8882 - dense_2_loss: 0.3838 - dense_3_loss: 0.5044 - val_loss: 0.9325 - val_dense_2_loss: 0.4092 - val_dense_3_loss: 0.5233
Epoch 41/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8846 - dense_2_loss: 0.3831 - dense_3_loss: 0.5016 - val_loss: 0.9277 - val_dense_2_loss: 0.4055 - val_dense_3_loss: 0.5221
Epoch 42/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8831 - dense_2_loss: 0.3827 - dense_3_loss: 0.5002 - val_loss: 0.9259 - val_dense_2_loss: 0.4053 - val_dense_3_loss: 0.5205
Epoch 43/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8804 - dense_2_loss: 0.3817 - dense_3_loss: 0.4987 - val_loss: 0.9199 - val_dense_2_loss: 0.4035 - val_dense_3_loss: 0.5163
Epoch 44/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8737 - dense_2_loss: 0.3792 - dense_3_loss: 0.4946 - val_loss: 0.9165 - val_dense_2_loss: 0.4009 - val_dense_3_loss: 0.5154
Epoch 45/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8698 - dense_2_loss: 0.3776 - dense_3_loss: 0.4920 - val_loss: 0.9099 - val_dense_2_loss: 0.3978 - val_dense_3_loss: 0.5120
Epoch 46/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8689 - dense_2_loss: 0.3779 - dense_3_loss: 0.4910 - val_loss: 0.9068 - val_dense_2_loss: 0.3983 - val_dense_3_loss: 0.5084
Epoch 47/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8622 - dense_2_loss: 0.3751 - dense_3_loss: 0.4870 - val_loss: 0.9008 - val_dense_2_loss: 0.3938 - val_dense_3_loss: 0.5069
Epoch 48/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8588 - dense_2_loss: 0.3739 - dense_3_loss: 0.4848 - val_loss: 0.8974 - val_dense_2_loss: 0.3932 - val_dense_3_loss: 0.5041
Epoch 49/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8582 - dense_2_loss: 0.3740 - dense_3_loss: 0.4840 - val_loss: 0.8923 - val_dense_2_loss: 0.3919 - val_dense_3_loss: 0.5003
Epoch 50/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8522 - dense_2_loss: 0.3718 - dense_3_loss: 0.4803 - val_loss: 0.8924 - val_dense_2_loss: 0.3922 - val_dense_3_loss: 0.5000
Epoch 51/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.8503 - dense_2_loss: 0.3715 - dense_3_loss: 0.4789 - val_loss: 0.8888 - val_dense_2_loss: 0.3900 - val_dense_3_loss: 0.4986
Epoch 52/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.8487 - dense_2_loss: 0.3714 - dense_3_loss: 0.4772 - val_loss: 0.8834 - val_dense_2_loss: 0.3910 - val_dense_3_loss: 0.4923
Epoch 53/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8443 - dense_2_loss: 0.3696 - dense_3_loss: 0.4746 - val_loss: 0.8799 - val_dense_2_loss: 0.3882 - val_dense_3_loss: 0.4915
Epoch 54/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8423 - dense_2_loss: 0.3691 - dense_3_loss: 0.4731 - val_loss: 0.8745 - val_dense_2_loss: 0.3870 - val_dense_3_loss: 0.4874
Epoch 55/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8385 - dense_2_loss: 0.3679 - dense_3_loss: 0.4705 - val_loss: 0.8729 - val_dense_2_loss: 0.3859 - val_dense_3_loss: 0.4870
Epoch 56/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8307 - dense_2_loss: 0.3647 - dense_3_loss: 0.4659 - val_loss: 0.8685 - val_dense_2_loss: 0.3823 - val_dense_3_loss: 0.4861
Epoch 57/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.8316 - dense_2_loss: 0.3654 - dense_3_loss: 0.4663 - val_loss: 0.8642 - val_dense_2_loss: 0.3818 - val_dense_3_loss: 0.4823
Epoch 58/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8277 - dense_2_loss: 0.3640 - dense_3_loss: 0.4636 - val_loss: 0.8592 - val_dense_2_loss: 0.3810 - val_dense_3_loss: 0.4781
Epoch 59/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8323 - dense_2_loss: 0.3661 - dense_3_loss: 0.4661 - val_loss: 0.8578 - val_dense_2_loss: 0.3809 - val_dense_3_loss: 0.4768
Epoch 60/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8185 - dense_2_loss: 0.3608 - dense_3_loss: 0.4580 - val_loss: 0.8512 - val_dense_2_loss: 0.3777 - val_dense_3_loss: 0.4734
Epoch 61/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8188 - dense_2_loss: 0.3609 - dense_3_loss: 0.4577 - val_loss: 0.8514 - val_dense_2_loss: 0.3779 - val_dense_3_loss: 0.4734
Epoch 62/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8165 - dense_2_loss: 0.3608 - dense_3_loss: 0.4556 - val_loss: 0.8442 - val_dense_2_loss: 0.3746 - val_dense_3_loss: 0.4695
Epoch 63/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.8118 - dense_2_loss: 0.3589 - dense_3_loss: 0.4530 - val_loss: 0.8407 - val_dense_2_loss: 0.3724 - val_dense_3_loss: 0.4682
Epoch 64/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8097 - dense_2_loss: 0.3587 - dense_3_loss: 0.4512 - val_loss: 0.8400 - val_dense_2_loss: 0.3758 - val_dense_3_loss: 0.4640
Epoch 65/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8025 - dense_2_loss: 0.3552 - dense_3_loss: 0.4471 - val_loss: 0.8353 - val_dense_2_loss: 0.3721 - val_dense_3_loss: 0.4631
Epoch 66/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7997 - dense_2_loss: 0.3542 - dense_3_loss: 0.4456 - val_loss: 0.8299 - val_dense_2_loss: 0.3707 - val_dense_3_loss: 0.4591
Epoch 67/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8013 - dense_2_loss: 0.3553 - dense_3_loss: 0.4458 - val_loss: 0.8289 - val_dense_2_loss: 0.3691 - val_dense_3_loss: 0.4597
Epoch 68/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7985 - dense_2_loss: 0.3548 - dense_3_loss: 0.4436 - val_loss: 0.8209 - val_dense_2_loss: 0.3677 - val_dense_3_loss: 0.4532
Epoch 69/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7968 - dense_2_loss: 0.3538 - dense_3_loss: 0.4428 - val_loss: 0.8211 - val_dense_2_loss: 0.3682 - val_dense_3_loss: 0.4527
Epoch 70/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7889 - dense_2_loss: 0.3510 - dense_3_loss: 0.4378 - val_loss: 0.8174 - val_dense_2_loss: 0.3667 - val_dense_3_loss: 0.4506
Epoch 71/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7877 - dense_2_loss: 0.3506 - dense_3_loss: 0.4373 - val_loss: 0.8115 - val_dense_2_loss: 0.3632 - val_dense_3_loss: 0.4482
Epoch 72/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7845 - dense_2_loss: 0.3496 - dense_3_loss: 0.4354 - val_loss: 0.8157 - val_dense_2_loss: 0.3665 - val_dense_3_loss: 0.4490
Epoch 73/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7818 - dense_2_loss: 0.3487 - dense_3_loss: 0.4332 - val_loss: 0.8084 - val_dense_2_loss: 0.3622 - val_dense_3_loss: 0.4461
Epoch 74/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.7843 - dense_2_loss: 0.3501 - dense_3_loss: 0.4348 - val_loss: 0.8084 - val_dense_2_loss: 0.3620 - val_dense_3_loss: 0.4463
Epoch 75/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7785 - dense_2_loss: 0.3474 - dense_3_loss: 0.4311 - val_loss: 0.8079 - val_dense_2_loss: 0.3622 - val_dense_3_loss: 0.4457
Epoch 76/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.7765 - dense_2_loss: 0.3470 - dense_3_loss: 0.4295 - val_loss: 0.8049 - val_dense_2_loss: 0.3639 - val_dense_3_loss: 0.4409
Epoch 77/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7735 - dense_2_loss: 0.3460 - dense_3_loss: 0.4276 - val_loss: 0.8012 - val_dense_2_loss: 0.3612 - val_dense_3_loss: 0.4399
Epoch 78/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7730 - dense_2_loss: 0.3459 - dense_3_loss: 0.4273 - val_loss: 0.8023 - val_dense_2_loss: 0.3603 - val_dense_3_loss: 0.4419
Epoch 79/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7756 - dense_2_loss: 0.3472 - dense_3_loss: 0.4284 - val_loss: 0.7986 - val_dense_2_loss: 0.3576 - val_dense_3_loss: 0.4409
Epoch 80/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7761 - dense_2_loss: 0.3475 - dense_3_loss: 0.4286 - val_loss: 0.7979 - val_dense_2_loss: 0.3598 - val_dense_3_loss: 0.4380
Epoch 81/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7733 - dense_2_loss: 0.3466 - dense_3_loss: 0.4267 - val_loss: 0.7936 - val_dense_2_loss: 0.3593 - val_dense_3_loss: 0.4341
Epoch 82/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7655 - dense_2_loss: 0.3430 - dense_3_loss: 0.4224 - val_loss: 0.7945 - val_dense_2_loss: 0.3580 - val_dense_3_loss: 0.4363
Epoch 83/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7669 - dense_2_loss: 0.3442 - dense_3_loss: 0.4224 - val_loss: 0.7913 - val_dense_2_loss: 0.3572 - val_dense_3_loss: 0.4340
Epoch 84/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7617 - dense_2_loss: 0.3424 - dense_3_loss: 0.4194 - val_loss: 0.7889 - val_dense_2_loss: 0.3561 - val_dense_3_loss: 0.4326
Epoch 85/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7601 - dense_2_loss: 0.3414 - dense_3_loss: 0.4186 - val_loss: 0.7866 - val_dense_2_loss: 0.3560 - val_dense_3_loss: 0.4304
Epoch 86/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7578 - dense_2_loss: 0.3408 - dense_3_loss: 0.4169 - val_loss: 0.7873 - val_dense_2_loss: 0.3562 - val_dense_3_loss: 0.4309
Epoch 87/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.7631 - dense_2_loss: 0.3432 - dense_3_loss: 0.4198 - val_loss: 0.7842 - val_dense_2_loss: 0.3540 - val_dense_3_loss: 0.4302
Epoch 88/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7603 - dense_2_loss: 0.3422 - dense_3_loss: 0.4181 - val_loss: 0.7832 - val_dense_2_loss: 0.3549 - val_dense_3_loss: 0.4282
Epoch 89/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7542 - dense_2_loss: 0.3391 - dense_3_loss: 0.4148 - val_loss: 0.7893 - val_dense_2_loss: 0.3571 - val_dense_3_loss: 0.4320
Epoch 90/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7528 - dense_2_loss: 0.3391 - dense_3_loss: 0.4139 - val_loss: 0.7808 - val_dense_2_loss: 0.3533 - val_dense_3_loss: 0.4274
Epoch 91/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7523 - dense_2_loss: 0.3385 - dense_3_loss: 0.4137 - val_loss: 0.7778 - val_dense_2_loss: 0.3532 - val_dense_3_loss: 0.4245
Epoch 92/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7514 - dense_2_loss: 0.3390 - dense_3_loss: 0.4130 - val_loss: 0.7748 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4241
Epoch 93/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.7479 - dense_2_loss: 0.3371 - dense_3_loss: 0.4106 - val_loss: 0.7771 - val_dense_2_loss: 0.3529 - val_dense_3_loss: 0.4241
Epoch 94/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7477 - dense_2_loss: 0.3371 - dense_3_loss: 0.4105 - val_loss: 0.7711 - val_dense_2_loss: 0.3500 - val_dense_3_loss: 0.4210
Epoch 95/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7519 - dense_2_loss: 0.3391 - dense_3_loss: 0.4126 - val_loss: 0.7720 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4212
Epoch 96/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7544 - dense_2_loss: 0.3404 - dense_3_loss: 0.4140 - val_loss: 0.7853 - val_dense_2_loss: 0.3559 - val_dense_3_loss: 0.4293
Epoch 97/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.7543 - dense_2_loss: 0.3404 - dense_3_loss: 0.4140 - val_loss: 0.7712 - val_dense_2_loss: 0.3506 - val_dense_3_loss: 0.4205
Epoch 98/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.7540 - dense_2_loss: 0.3405 - dense_3_loss: 0.4140 - val_loss: 0.7728 - val_dense_2_loss: 0.3520 - val_dense_3_loss: 0.4207
Epoch 99/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.7424 - dense_2_loss: 0.3352 - dense_3_loss: 0.4070 - val_loss: 0.7710 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4203
5160/1 [====================================================] - 0s 29us/sample - loss: 0.8831 - dense_2_loss: 0.3513 - dense_3_loss: 0.4160
[0.7661929805149403, 0.35130394, 0.41597202]

這裡會有三組線，一個是總的，一個是第一個輸出，一個是第二個輸出。

基礎API

我們現在來看一下tensorflow中最基本的常量操作

import tensorflow as tf
import numpy as np

if __name__ == "__main__":

    # 定義矩陣常量
    t = tf.constant([[1., 2., 3.], [4., 5., 6.]])
    print(t)
    print(t[:, 1:])
    print(t[:, 1])
    # 運算元
    print(t + 10)
    # 計算平方
    print(tf.square(t))
    # 計算它本身和它轉置的點乘
    print(t @ tf.transpose(t))
    # 獲取numpy的值
    print(t.numpy())
    # 作為numpy的輸入
    print(np.square(t))
    # 將numpy物件轉成tensorflow物件
    np_t = np.array([[1., 2., 3.], [4., 5., 6.]])
    print(tf.constant(np_t))
    # 標量
    num = tf.constant(2.718)
    print(num)
    print(num.numpy())
    print(num.shape)
    # 字串
    t = tf.constant("coffee")
    print(t)
    print(tf.strings.length(t))
    print(tf.strings.length(t, unit='UTF8_CHAR'))
    # 解碼成utf8編碼
    print(tf.strings.unicode_decode(t, 'UTF8'))
    # 字串陣列
    t = tf.constant(['cafe', 'coffee', '咖啡'])
    print(t)
    print(tf.strings.length(t, unit='UTF8_CHAR'))
    print(tf.strings.unicode_decode(t, 'UTF8'))
    # ragged tensor不等長的陣列
    r = tf.ragged.constant([[11, 12], [21, 22, 23], [], [41]])
    print(r)
    print(r[1])
    print(r[1:3])
    # ragged tensor運算元
    r2 = tf.ragged.constant([[51, 52], [], [71], [42, 43]])
    # 在第0個維度上進行拼接
    print(tf.concat([r, r2], axis=0))
    # 在第1個維度上進行拼接,如果第一個維度的數量不相等，會報錯
    print(tf.concat([r, r2], axis=1))
    # 將ragged tensor轉化為普通tensor，空缺位置補0
    print(r.to_tensor())
    # sparse tensor 大部分位置的值為0，只有少部分不為0
    # indices必須排序放置，否則轉普通tensor會報錯
    s = tf.SparseTensor(indices=[[0, 1], [1, 0], [2, 3]],
                        values=[1., 2., 3], dense_shape=[3, 4])
    print(s)
    # 將sparse tensor轉化為普通的tensor
    print(tf.sparse.to_dense(s))
    # sparse tensor運算元
    s2 = s * 2.0
    print(s2)
    try:
        s3 = s + 1.
    except TypeError as ex:
        print(ex)
    s4 = tf.constant([[10., 20.], [30., 40.], [50., 60.], [70., 80.]])
    # 矩陣乘法
    print(tf.sparse.sparse_dense_matmul(s, s4))
    s5 = tf.SparseTensor(indices=[[0, 2], [0, 1], [2, 3]],
                         values=[1., 2., 3], dense_shape=[3, 4])
    print(s5)
    try:
        print(tf.sparse.to_dense(s5))
    except Exception as ex:
        print(ex)
    s6 = tf.sparse.reorder(s5)
    print(tf.sparse.to_dense(s6))

執行結果

tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[2. 3.]
 [5. 6.]], shape=(2, 2), dtype=float32)
tf.Tensor([2. 5.], shape=(2,), dtype=float32)
tf.Tensor(
[[11. 12. 13.]
 [14. 15. 16.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[ 1.  4.  9.]
 [16. 25. 36.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[14. 32.]
 [32. 77.]], shape=(2, 2), dtype=float32)
[[1. 2. 3.]
 [4. 5. 6.]]
[[ 1.  4.  9.]
 [16. 25. 36.]]
tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float64)
tf.Tensor(2.718, shape=(), dtype=float32)
2.718
()
tf.Tensor(b'coffee', shape=(), dtype=string)
tf.Tensor(6, shape=(), dtype=int32)
tf.Tensor(6, shape=(), dtype=int32)
tf.Tensor([ 99 111 102 102 101 101], shape=(6,), dtype=int32)
tf.Tensor([b'cafe' b'coffee' b'\xe5\x92\x96\xe5\x95\xa1'], shape=(3,), dtype=string)
tf.Tensor([4 6 2], shape=(3,), dtype=int32)
<tf.RaggedTensor [[99, 97, 102, 101], [99, 111, 102, 102, 101, 101], [21654, 21857]]>
<tf.RaggedTensor [[11, 12], [21, 22, 23], [], [41]]>
tf.Tensor([21 22 23], shape=(3,), dtype=int32)
<tf.RaggedTensor [[21, 22, 23], []]>
<tf.RaggedTensor [[11, 12], [21, 22, 23], [], [41], [51, 52], [], [71], [42, 43]]>
<tf.RaggedTensor [[11, 12, 51, 52], [21, 22, 23], [71], [41, 42, 43]]>
tf.Tensor(
[[11 12  0]
 [21 22 23]
 [ 0  0  0]
 [41  0  0]], shape=(4, 3), dtype=int32)
SparseTensor(indices=tf.Tensor(
[[0 1]
 [1 0]
 [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))
tf.Tensor(
[[0. 1. 0. 0.]
 [2. 0. 0. 0.]
 [0. 0. 0. 3.]], shape=(3, 4), dtype=float32)
SparseTensor(indices=tf.Tensor(
[[0 1]
 [1 0]
 [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([2. 4. 6.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))
unsupported operand type(s) for +: 'SparseTensor' and 'float'
tf.Tensor(
[[ 30.  40.]
 [ 20.  40.]
 [210. 240.]], shape=(3, 2), dtype=float32)
SparseTensor(indices=tf.Tensor(
[[0 2]
 [0 1]
 [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))
indices[1] = [0,1] is out of order [Op:SparseToDense]
tf.Tensor(
[[0. 2. 1. 0.]
 [0. 0. 0. 0.]
 [0. 0. 0. 3.]], shape=(3, 4), dtype=float32)

現在我們來看一下它的變數操作

import tensorflow as tf
import numpy as np

if __name__ == "__main__":

    v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])
    print(v)
    # 轉化為常量
    print(v.value())
    print(v.numpy())
    # 重新賦值
    v.assign(2 * v)
    print(v)
    # 給變數的某個位置重新賦值
    v[0, 1].assign(42)
    print(v)
    # 給變數的某一行重新賦值
    v[1].assign([7., 8., 9.])
    print(v)
    # 不能直接使用=賦值
    try:
        v[0] = [10., 20., 30.]
    except TypeError as ex:
        print(ex)

執行結果

<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[1., 2., 3.],
       [4., 5., 6.]], dtype=float32)>
tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float32)
[[1. 2. 3.]
 [4. 5. 6.]]
<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[ 2.,  4.,  6.],
       [ 8., 10., 12.]], dtype=float32)>
<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[ 2., 42.,  6.],
       [ 8., 10., 12.]], dtype=float32)>
<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[ 2., 42.,  6.],
       [ 7.,  8.,  9.]], dtype=float32)>
'ResourceVariable' object does not support item assignment

自定義損失函式

這裡我們還是以加利福尼亞的房價迴歸問題來說明

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)

    def customized_mse(y_true, y_pred):
        # 自定義損失函式
        return tf.reduce_mean(tf.square(y_pred - y_true))

    model = keras.models.Sequential([
        layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
        layers.Dense(1)
    ])
    print(model.summary())
    # 使用均方誤差損失函式和隨機梯度下降法
    model.compile(loss=customized_mse, optimizer=optimizers.SGD(learning_rate=0.01),
                  metrics=['mean_squared_error'])
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, 30)                270       
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 31        
=================================================================
Total params: 301
Trainable params: 301
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 119us/sample - loss: 1.5018 - mean_squared_error: 1.5018 - val_loss: 0.8721 - val_mean_squared_error: 0.8721
Epoch 2/100
11610/11610 [==============================] - 0s 43us/sample - loss: 1.8099 - mean_squared_error: 1.8099 - val_loss: 0.5967 - val_mean_squared_error: 0.5967
Epoch 3/100
11610/11610 [==============================] - 1s 43us/sample - loss: 0.4971 - mean_squared_error: 0.4971 - val_loss: 0.4621 - val_mean_squared_error: 0.4621
Epoch 4/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.4396 - mean_squared_error: 0.4396 - val_loss: 0.4405 - val_mean_squared_error: 0.4405
Epoch 5/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.4121 - mean_squared_error: 0.4121 - val_loss: 0.4133 - val_mean_squared_error: 0.4133
Epoch 6/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.4043 - mean_squared_error: 0.4043 - val_loss: 0.4039 - val_mean_squared_error: 0.4039
Epoch 7/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3896 - mean_squared_error: 0.3896 - val_loss: 0.4000 - val_mean_squared_error: 0.4000
Epoch 8/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3796 - mean_squared_error: 0.3796 - val_loss: 0.3936 - val_mean_squared_error: 0.3936
Epoch 9/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3771 - mean_squared_error: 0.3771 - val_loss: 0.5541 - val_mean_squared_error: 0.5541
Epoch 10/100
11610/11610 [==============================] - 1s 46us/sample - loss: 0.3820 - mean_squared_error: 0.3820 - val_loss: 0.3874 - val_mean_squared_error: 0.3874
Epoch 11/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3686 - mean_squared_error: 0.3686 - val_loss: 0.3758 - val_mean_squared_error: 0.3758
Epoch 12/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3666 - mean_squared_error: 0.3666 - val_loss: 0.3867 - val_mean_squared_error: 0.3867
Epoch 13/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3632 - mean_squared_error: 0.3632 - val_loss: 0.3711 - val_mean_squared_error: 0.3711
Epoch 14/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3592 - mean_squared_error: 0.3592 - val_loss: 0.3790 - val_mean_squared_error: 0.3790
Epoch 15/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3578 - mean_squared_error: 0.3578 - val_loss: 0.3687 - val_mean_squared_error: 0.3687
Epoch 16/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3547 - mean_squared_error: 0.3547 - val_loss: 0.3691 - val_mean_squared_error: 0.3691
Epoch 17/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3530 - mean_squared_error: 0.3530 - val_loss: 0.3690 - val_mean_squared_error: 0.3690
Epoch 18/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.3549 - mean_squared_error: 0.3549 - val_loss: 0.3625 - val_mean_squared_error: 0.3625
Epoch 19/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.3506 - mean_squared_error: 0.3506 - val_loss: 0.3605 - val_mean_squared_error: 0.3605
Epoch 20/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3481 - mean_squared_error: 0.3481 - val_loss: 0.3610 - val_mean_squared_error: 0.3610
Epoch 21/100
11610/11610 [==============================] - 1s 72us/sample - loss: 0.3479 - mean_squared_error: 0.3479 - val_loss: 0.3601 - val_mean_squared_error: 0.3601
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3448 - mean_squared_error: 0.3448 - val_loss: 0.3578 - val_mean_squared_error: 0.3578
Epoch 23/100
11610/11610 [==============================] - 1s 73us/sample - loss: 0.3457 - mean_squared_error: 0.3457 - val_loss: 0.3611 - val_mean_squared_error: 0.3611
Epoch 24/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3447 - mean_squared_error: 0.3447 - val_loss: 0.3539 - val_mean_squared_error: 0.3539
Epoch 25/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3432 - mean_squared_error: 0.3432 - val_loss: 0.3515 - val_mean_squared_error: 0.3515
Epoch 26/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3404 - mean_squared_error: 0.3404 - val_loss: 0.3544 - val_mean_squared_error: 0.3544
Epoch 27/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3406 - mean_squared_error: 0.3406 - val_loss: 0.3504 - val_mean_squared_error: 0.3504
Epoch 28/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3389 - mean_squared_error: 0.3389 - val_loss: 0.3514 - val_mean_squared_error: 0.3514
Epoch 29/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3385 - mean_squared_error: 0.3385 - val_loss: 0.3470 - val_mean_squared_error: 0.3470
Epoch 30/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3375 - mean_squared_error: 0.3375 - val_loss: 0.3513 - val_mean_squared_error: 0.3513
Epoch 31/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3362 - mean_squared_error: 0.3362 - val_loss: 0.3474 - val_mean_squared_error: 0.3474
Epoch 32/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3351 - mean_squared_error: 0.3351 - val_loss: 0.3493 - val_mean_squared_error: 0.3493
Epoch 33/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3349 - mean_squared_error: 0.3349 - val_loss: 0.3461 - val_mean_squared_error: 0.3461
Epoch 34/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3326 - mean_squared_error: 0.3326 - val_loss: 0.3464 - val_mean_squared_error: 0.3464
5160/1 [=====================================================] - 0s 20us/sample - loss: 0.5104 - mean_squared_error: 0.3525
[0.3524518048116403, 0.3524517]

這裡我們可以看到我們自己實現的均方誤差損失函式和系統自帶的均方誤差損失函式是一樣，他們損失函式值曲線是重合的。

自定義Layers

我們先來看一下Layers的一些特性

import tensorflow as tf
from tensorflow.keras import layers

if __name__ == "__main__":

    # 我們一般會在全連線層的第一層的時候輸入input_shape
    # 不使用input_shape也沒關係，它會在第一次執行的時候推匯出來
    layer = layers.Dense(100, input_shape=(None, 5))
    # 這裡會點乘一個[5, 100]的矩陣轉成[None, 100]的輸出
    print(layer(tf.zeros([10, 5])))
    # 列印該層所有的引數
    print(layer.variables)
    # 列印該層所有可訓練的引數
    print(layer.trainable_variables)

執行結果

tf.Tensor(
[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]], shape=(10, 100), dtype=float32)
[<tf.Variable 'dense/kernel:0' shape=(5, 100) dtype=float32, numpy=
array([[ 0.17777802, -0.0925342 ,  0.22044905, -0.1196067 , -0.15553904,
        -0.19373423, -0.13884991, -0.23252396, -0.01252402, -0.09044193,
        -0.06529157,  0.22907178, -0.1411261 ,  0.20145597,  0.11501376,
        -0.15268266, -0.17305377, -0.01125969, -0.10907765,  0.0403709 ,
         0.06923743, -0.00155647, -0.18127626,  0.11550726,  0.03372289,
         0.13893865, -0.0092984 , -0.20496821, -0.02627921,  0.01125975,
        -0.04647307,  0.15463649,  0.17841218,  0.17077957, -0.0842554 ,
         0.0047874 , -0.12141771,  0.22519882,  0.23548423,  0.06460319,
        -0.03109986, -0.12173892, -0.02452356, -0.04257253,  0.2064407 ,
        -0.06991957,  0.06425266, -0.1007075 ,  0.19131865, -0.1955776 ,
         0.21193884, -0.2004574 , -0.07692617, -0.08107474,  0.1934263 ,
         0.05712549,  0.01886164, -0.12672797, -0.17843167, -0.05964956,
        -0.238883  , -0.21325174,  0.11310308, -0.00318795, -0.12962502,
        -0.16521364,  0.12938736, -0.23181748,  0.21444045, -0.12927116,
        -0.18740693,  0.05530755,  0.0146782 ,  0.21035798, -0.15627733,
        -0.01847358, -0.08605188, -0.0206164 ,  0.20378043, -0.03834918,
         0.0639727 , -0.06449054,  0.17925756, -0.14535832,  0.22247057,
         0.08074225,  0.19761471, -0.20858642,  0.17465816, -0.12256925,
        -0.07192959, -0.01034951, -0.2167171 ,  0.02262951, -0.2061531 ,
         0.19094323, -0.0974756 ,  0.21508281, -0.01393197,  0.19384028],
       [-0.02444541, -0.21043453,  0.06121971, -0.18661837, -0.1679871 ,
         0.17915292,  0.0956945 , -0.03815825, -0.10663904, -0.04658341,
        -0.18815587, -0.04888278,  0.19361894, -0.13932027, -0.2013808 ,
        -0.2031331 , -0.12357853,  0.15468134, -0.16377276, -0.15836287,
        -0.03290932,  0.09049819, -0.10049133, -0.1802709 ,  0.11397789,
        -0.22433408, -0.01369557, -0.21804577,  0.13998653, -0.0452639 ,
        -0.10171422, -0.15374628, -0.07929   ,  0.08984323,  0.11814152,
         0.13589476,  0.05618854,  0.1663635 , -0.01764336,  0.09068643,
        -0.1157929 , -0.1266903 , -0.07107972,  0.15617536, -0.1517889 ,
        -0.21622056, -0.17632967,  0.14407007,  0.10903122, -0.00500569,
         0.05290176, -0.03641239, -0.2377142 , -0.12242448, -0.07954802,
         0.09190597, -0.09879458, -0.18804912, -0.14522013,  0.07479332,
         0.17024298,  0.0161783 , -0.0524762 ,  0.0176069 , -0.08270177,
         0.16904478,  0.17900594,  0.15703659,  0.19818316, -0.1625318 ,
        -0.12794688, -0.1563586 , -0.09517953,  0.09158082,  0.23767962,
         0.20551957,  0.14481695,  0.2045602 , -0.06479037,  0.05358954,
         0.1212578 , -0.02165481, -0.1284295 , -0.08271579, -0.05446866,
         0.05413379,  0.00234202,  0.22771172, -0.2224471 , -0.03084002,
        -0.19375737,  0.0798779 , -0.09207922,  0.13679741, -0.1429534 ,
         0.0209298 , -0.0841022 , -0.16638935,  0.0903822 , -0.19967408],
       [ 0.11377819, -0.03955515,  0.02814175,  0.04407446,  0.02015819,
        -0.01894131,  0.14453827,  0.07663743, -0.2206468 , -0.23865755,
        -0.23881991,  0.01236854, -0.09480104,  0.20786951, -0.02678156,
        -0.0305012 , -0.05566147, -0.21860714, -0.1938608 ,  0.053019  ,
        -0.05170445, -0.1206295 , -0.22782467,  0.06737016,  0.05737154,
         0.01793937, -0.2030854 , -0.14422959, -0.17374498, -0.02698712,
        -0.13331282,  0.12249409, -0.19247851,  0.14200823, -0.16609323,
         0.14399786,  0.21913432, -0.21645993, -0.191912  ,  0.0571617 ,
         0.19024946,  0.08669983,  0.08364411, -0.1892563 , -0.02523425,
         0.11180784,  0.00898317,  0.03620596, -0.08056329,  0.19991381,
         0.11432077,  0.1168678 , -0.1644195 ,  0.00769468,  0.22227935,
        -0.20849067,  0.15359052,  0.1345963 , -0.21721914,  0.0614645 ,
         0.12685539,  0.11890687, -0.20878732, -0.15853551,  0.0458544 ,
        -0.15758611, -0.16187118, -0.09367229, -0.16155185, -0.18485507,
        -0.13647649, -0.12254263, -0.05565156, -0.18738703, -0.06917199,
        -0.03574945,  0.17475249, -0.17476122, -0.10158296,  0.12375195,
         0.18355547,  0.03811152,  0.07888819,  0.09909324, -0.15394625,
         0.11169995, -0.09254411, -0.10357954,  0.19252472,  0.02573626,
        -0.01318276, -0.09475978, -0.1874548 , -0.19183636, -0.18057513,
        -0.15745679,  0.18442439,  0.08096658, -0.01627833,  0.05583473],
       [-0.04097484,  0.23634328, -0.1889084 ,  0.08590437,  0.17894466,
        -0.11124423,  0.18041344, -0.21274638,  0.11614217,  0.10249467,
         0.21891718,  0.19230996,  0.15266438, -0.10032661, -0.22356763,
         0.11601089, -0.08055103,  0.00928256,  0.16465248, -0.04512097,
        -0.17905587, -0.23013544, -0.00562099, -0.09802979, -0.22574061,
         0.02589588,  0.19565006,  0.09272386, -0.13209037, -0.21364078,
        -0.15901634,  0.21333267, -0.22375406, -0.17148429,  0.15592472,
        -0.1618546 ,  0.20303024,  0.04697676,  0.20634227, -0.1878765 ,
         0.22583528, -0.12808219,  0.18875875, -0.19559893,  0.15292238,
         0.08833222,  0.1192212 ,  0.12659721,  0.12049772, -0.21185097,
         0.16098978, -0.11107406, -0.13046448,  0.21239029,  0.00350848,
         0.15068768, -0.0189786 ,  0.1369033 , -0.10414104,  0.16339312,
        -0.09338254, -0.14816928, -0.2035834 ,  0.16379024,  0.01705651,
         0.11921181,  0.23129468,  0.18642814, -0.17961605,  0.22902219,
         0.2109852 , -0.19536257,  0.06335585,  0.05963777,  0.0976281 ,
         0.00718886,  0.02180146,  0.16579719, -0.23447831, -0.22135682,
        -0.15220946, -0.03425349, -0.16059369, -0.17343065,  0.12119611,
         0.07790975, -0.07582501, -0.03742242,  0.18471824,  0.20959799,
        -0.22154935, -0.2013107 , -0.12604958,  0.12774341, -0.01246983,
        -0.18907636,  0.03709231, -0.02248657,  0.03133444, -0.1615734 ],
       [-0.16601531,  0.21792419, -0.1103511 ,  0.0025705 , -0.05631484,
         0.07372896,  0.18572135, -0.07337801, -0.01120584, -0.13398278,
        -0.04314138,  0.17321984,  0.11909981, -0.15493754,  0.07232957,
         0.13085563, -0.2287109 , -0.21197955,  0.1428767 ,  0.1137528 ,
        -0.14559013,  0.13985835,  0.12416814, -0.05637537, -0.0046737 ,
         0.00876671,  0.05863507, -0.03590567,  0.10624601, -0.1412015 ,
         0.15561827,  0.10837926, -0.1810194 , -0.05347614,  0.03898682,
        -0.10682511, -0.16920623, -0.18795235, -0.08359696,  0.18185644,
        -0.19991723, -0.05360089,  0.13864844,  0.1414309 , -0.23352294,
         0.21933769, -0.2087497 ,  0.02414815, -0.20864718, -0.01974651,
        -0.12589416, -0.04501775,  0.09451391,  0.01687138,  0.13819186,
        -0.21920864,  0.22943564,  0.22025959,  0.07460229, -0.11851113,
         0.04449643, -0.11150999,  0.18484862,  0.18422444, -0.19684097,
         0.125441  ,  0.16443445,  0.174614  ,  0.21528427, -0.02458197,
         0.2311335 , -0.20051023,  0.16148575,  0.03468569,  0.08386306,
         0.18408652, -0.09470819, -0.22607385, -0.04776055,  0.17687435,
         0.01153742,  0.15740515,  0.22579281,  0.2256033 ,  0.23760812,
        -0.08624862, -0.04818098, -0.09573658,  0.00707476, -0.174594  ,
         0.17612858, -0.02651523, -0.03352439, -0.0982894 ,  0.12405153,
         0.1423658 ,  0.12438641, -0.22709328,  0.18641399, -0.0159114 ]],
      dtype=float32)>, <tf.Variable 'dense/bias:0' shape=(100,) dtype=float32, numpy=
array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
      dtype=float32)>]
[<tf.Variable 'dense/kernel:0' shape=(5, 100) dtype=float32, numpy=
array([[ 0.17777802, -0.0925342 ,  0.22044905, -0.1196067 , -0.15553904,
        -0.19373423, -0.13884991, -0.23252396, -0.01252402, -0.09044193,
        -0.06529157,  0.22907178, -0.1411261 ,  0.20145597,  0.11501376,
        -0.15268266, -0.17305377, -0.01125969, -0.10907765,  0.0403709 ,
         0.06923743, -0.00155647, -0.18127626,  0.11550726,  0.03372289,
         0.13893865, -0.0092984 , -0.20496821, -0.02627921,  0.01125975,
        -0.04647307,  0.15463649,  0.17841218,  0.17077957, -0.0842554 ,
         0.0047874 , -0.12141771,  0.22519882,  0.23548423,  0.06460319,
        -0.03109986, -0.12173892, -0.02452356, -0.04257253,  0.2064407 ,
        -0.06991957,  0.06425266, -0.1007075 ,  0.19131865, -0.1955776 ,
         0.21193884, -0.2004574 , -0.07692617, -0.08107474,  0.1934263 ,
         0.05712549,  0.01886164, -0.12672797, -0.17843167, -0.05964956,
        -0.238883  , -0.21325174,  0.11310308, -0.00318795, -0.12962502,
        -0.16521364,  0.12938736, -0.23181748,  0.21444045, -0.12927116,
        -0.18740693,  0.05530755,  0.0146782 ,  0.21035798, -0.15627733,
        -0.01847358, -0.08605188, -0.0206164 ,  0.20378043, -0.03834918,
         0.0639727 , -0.06449054,  0.17925756, -0.14535832,  0.22247057,
         0.08074225,  0.19761471, -0.20858642,  0.17465816, -0.12256925,
        -0.07192959, -0.01034951, -0.2167171 ,  0.02262951, -0.2061531 ,
         0.19094323, -0.0974756 ,  0.21508281, -0.01393197,  0.19384028],
       [-0.02444541, -0.21043453,  0.06121971, -0.18661837, -0.1679871 ,
         0.17915292,  0.0956945 , -0.03815825, -0.10663904, -0.04658341,
        -0.18815587, -0.04888278,  0.19361894, -0.13932027, -0.2013808 ,
        -0.2031331 , -0.12357853,  0.15468134, -0.16377276, -0.15836287,
        -0.03290932,  0.09049819, -0.10049133, -0.1802709 ,  0.11397789,
        -0.22433408, -0.01369557, -0.21804577,  0.13998653, -0.0452639 ,
        -0.10171422, -0.15374628, -0.07929   ,  0.08984323,  0.11814152,
         0.13589476,  0.05618854,  0.1663635 , -0.01764336,  0.09068643,
        -0.1157929 , -0.1266903 , -0.07107972,  0.15617536, -0.1517889 ,
        -0.21622056, -0.17632967,  0.14407007,  0.10903122, -0.00500569,
         0.05290176, -0.03641239, -0.2377142 , -0.12242448, -0.07954802,
         0.09190597, -0.09879458, -0.18804912, -0.14522013,  0.07479332,
         0.17024298,  0.0161783 , -0.0524762 ,  0.0176069 , -0.08270177,
         0.16904478,  0.17900594,  0.15703659,  0.19818316, -0.1625318 ,
        -0.12794688, -0.1563586 , -0.09517953,  0.09158082,  0.23767962,
         0.20551957,  0.14481695,  0.2045602 , -0.06479037,  0.05358954,
         0.1212578 , -0.02165481, -0.1284295 , -0.08271579, -0.05446866,
         0.05413379,  0.00234202,  0.22771172, -0.2224471 , -0.03084002,
        -0.19375737,  0.0798779 , -0.09207922,  0.13679741, -0.1429534 ,
         0.0209298 , -0.0841022 , -0.16638935,  0.0903822 , -0.19967408],
       [ 0.11377819, -0.03955515,  0.02814175,  0.04407446,  0.02015819,
        -0.01894131,  0.14453827,  0.07663743, -0.2206468 , -0.23865755,
        -0.23881991,  0.01236854, -0.09480104,  0.20786951, -0.02678156,
        -0.0305012 , -0.05566147, -0.21860714, -0.1938608 ,  0.053019  ,
        -0.05170445, -0.1206295 , -0.22782467,  0.06737016,  0.05737154,
         0.01793937, -0.2030854 , -0.14422959, -0.17374498, -0.02698712,
        -0.13331282,  0.12249409, -0.19247851,  0.14200823, -0.16609323,
         0.14399786,  0.21913432, -0.21645993, -0.191912  ,  0.0571617 ,
         0.19024946,  0.08669983,  0.08364411, -0.1892563 , -0.02523425,
         0.11180784,  0.00898317,  0.03620596, -0.08056329,  0.19991381,
         0.11432077,  0.1168678 , -0.1644195 ,  0.00769468,  0.22227935,
        -0.20849067,  0.15359052,  0.1345963 , -0.21721914,  0.0614645 ,
         0.12685539,  0.11890687, -0.20878732, -0.15853551,  0.0458544 ,
        -0.15758611, -0.16187118, -0.09367229, -0.16155185, -0.18485507,
        -0.13647649, -0.12254263, -0.05565156, -0.18738703, -0.06917199,
        -0.03574945,  0.17475249, -0.17476122, -0.10158296,  0.12375195,
         0.18355547,  0.03811152,  0.07888819,  0.09909324, -0.15394625,
         0.11169995, -0.09254411, -0.10357954,  0.19252472,  0.02573626,
        -0.01318276, -0.09475978, -0.1874548 , -0.19183636, -0.18057513,
        -0.15745679,  0.18442439,  0.08096658, -0.01627833,  0.05583473],
       [-0.04097484,  0.23634328, -0.1889084 ,  0.08590437,  0.17894466,
        -0.11124423,  0.18041344, -0.21274638,  0.11614217,  0.10249467,
         0.21891718,  0.19230996,  0.15266438, -0.10032661, -0.22356763,
         0.11601089, -0.08055103,  0.00928256,  0.16465248, -0.04512097,
        -0.17905587, -0.23013544, -0.00562099, -0.09802979, -0.22574061,
         0.02589588,  0.19565006,  0.09272386, -0.13209037, -0.21364078,
        -0.15901634,  0.21333267, -0.22375406, -0.17148429,  0.15592472,
        -0.1618546 ,  0.20303024,  0.04697676,  0.20634227, -0.1878765 ,
         0.22583528, -0.12808219,  0.18875875, -0.19559893,  0.15292238,
         0.08833222,  0.1192212 ,  0.12659721,  0.12049772, -0.21185097,
         0.16098978, -0.11107406, -0.13046448,  0.21239029,  0.00350848,
         0.15068768, -0.0189786 ,  0.1369033 , -0.10414104,  0.16339312,
        -0.09338254, -0.14816928, -0.2035834 ,  0.16379024,  0.01705651,
         0.11921181,  0.23129468,  0.18642814, -0.17961605,  0.22902219,
         0.2109852 , -0.19536257,  0.06335585,  0.05963777,  0.0976281 ,
         0.00718886,  0.02180146,  0.16579719, -0.23447831, -0.22135682,
        -0.15220946, -0.03425349, -0.16059369, -0.17343065,  0.12119611,
         0.07790975, -0.07582501, -0.03742242,  0.18471824,  0.20959799,
        -0.22154935, -0.2013107 , -0.12604958,  0.12774341, -0.01246983,
        -0.18907636,  0.03709231, -0.02248657,  0.03133444, -0.1615734 ],
       [-0.16601531,  0.21792419, -0.1103511 ,  0.0025705 , -0.05631484,
         0.07372896,  0.18572135, -0.07337801, -0.01120584, -0.13398278,
        -0.04314138,  0.17321984,  0.11909981, -0.15493754,  0.07232957,
         0.13085563, -0.2287109 , -0.21197955,  0.1428767 ,  0.1137528 ,
        -0.14559013,  0.13985835,  0.12416814, -0.05637537, -0.0046737 ,
         0.00876671,  0.05863507, -0.03590567,  0.10624601, -0.1412015 ,
         0.15561827,  0.10837926, -0.1810194 , -0.05347614,  0.03898682,
        -0.10682511, -0.16920623, -0.18795235, -0.08359696,  0.18185644,
        -0.19991723, -0.05360089,  0.13864844,  0.1414309 , -0.23352294,
         0.21933769, -0.2087497 ,  0.02414815, -0.20864718, -0.01974651,
        -0.12589416, -0.04501775,  0.09451391,  0.01687138,  0.13819186,
        -0.21920864,  0.22943564,  0.22025959,  0.07460229, -0.11851113,
         0.04449643, -0.11150999,  0.18484862,  0.18422444, -0.19684097,
         0.125441  ,  0.16443445,  0.174614  ,  0.21528427, -0.02458197,
         0.2311335 , -0.20051023,  0.16148575,  0.03468569,  0.08386306,
         0.18408652, -0.09470819, -0.22607385, -0.04776055,  0.17687435,
         0.01153742,  0.15740515,  0.22579281,  0.2256033 ,  0.23760812,
        -0.08624862, -0.04818098, -0.09573658,  0.00707476, -0.174594  ,
         0.17612858, -0.02651523, -0.03352439, -0.0982894 ,  0.12405153,
         0.1423658 ,  0.12438641, -0.22709328,  0.18641399, -0.0159114 ]],
      dtype=float32)>, <tf.Variable 'dense/bias:0' shape=(100,) dtype=float32, numpy=
array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
      dtype=float32)>]

通過結果我們可以看出layer的輸出結果是一個常量，而它的引數是兩個變數。其中dense/kernel:0就是我們經常說的線性模型Wx+b中的係數W，而dense/bias:0就是其中的截距b。而且在這裡它的引數和可訓練的引數是一致的。現在我們來實現自定義的Dense_Layer。

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)

    def customized_mse(y_true, y_pred):
        # 自定義損失函式
        return tf.reduce_mean(tf.square(y_pred - y_true))

    class CustomizedDenseLayer(layers.Layer):
        # 自定義DenseLayer

        def __init__(self, units, activation=None, **kwargs):
            '''
            :param units: 輸出大小
            :param activation: 啟用函式
            :param kwargs:
            '''
            self.units = units
            self.activation = layers.Activation(activation=activation)
            super(CustomizedDenseLayer, self).__init__(**kwargs)

        def build(self, input_shape):
            '''
            構建所需要的引數
            :param input_shape: 輸入的形狀
            :return:
            '''
            # Wx + b，input_shape:[None, a],W:[a, b],output_shape:[None, b]
            # shape為W的形狀，即變換矩陣
            # uniform表示均勻分佈的方式初始化
            # trainable = True表示引數是可以被訓練的
            self.kernel = self.add_weight(name='kernel',
                                          shape=(input_shape[1], self.units),
                                          initializer='uniform',
                                          trainable=True)
            self.bias = self.add_weight(name='bias',
                                        shape=(self.units,),
                                        initializer='zeros',
                                        trainable=True)
            super(CustomizedDenseLayer, self).build(input_shape)

        def call(self, X):
            # 完成正向計算
            return self.activation(X @ self.kernel + self.bias)

    model = keras.models.Sequential([
        CustomizedDenseLayer(30, activation='relu', input_shape=X_train.shape[1:]),
        CustomizedDenseLayer(1)
    ])
    print(model.summary())
    # 使用均方誤差損失函式和隨機梯度下降法
    model.compile(loss=customized_mse, optimizer=optimizers.SGD(learning_rate=0.01),
                  metrics=['mean_squared_error'])
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

執行結果

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
customized_dense_layer (Cust (None, 30)                270       
_________________________________________________________________
customized_dense_layer_1 (Cu (None, 1)                 31        
=================================================================
Total params: 301
Trainable params: 301
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 98us/sample - loss: 1.1060 - mean_squared_error: 1.1061 - val_loss: 0.6408 - val_mean_squared_error: 0.6408
Epoch 2/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.5543 - mean_squared_error: 0.5543 - val_loss: 0.5453 - val_mean_squared_error: 0.5453
Epoch 3/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4889 - mean_squared_error: 0.4889 - val_loss: 0.4974 - val_mean_squared_error: 0.4974
Epoch 4/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4619 - mean_squared_error: 0.4619 - val_loss: 0.4884 - val_mean_squared_error: 0.4884
Epoch 5/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4479 - mean_squared_error: 0.4479 - val_loss: 0.4610 - val_mean_squared_error: 0.4610
Epoch 6/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4337 - mean_squared_error: 0.4337 - val_loss: 0.4442 - val_mean_squared_error: 0.4442
Epoch 7/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4234 - mean_squared_error: 0.4234 - val_loss: 0.4362 - val_mean_squared_error: 0.4362
Epoch 8/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4155 - mean_squared_error: 0.4155 - val_loss: 0.4295 - val_mean_squared_error: 0.4295
Epoch 9/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4164 - mean_squared_error: 0.4164 - val_loss: 0.4230 - val_mean_squared_error: 0.4230
Epoch 10/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4067 - mean_squared_error: 0.4067 - val_loss: 0.4183 - val_mean_squared_error: 0.4183
Epoch 11/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3986 - mean_squared_error: 0.3986 - val_loss: 0.4144 - val_mean_squared_error: 0.4144
Epoch 12/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3946 - mean_squared_error: 0.3946 - val_loss: 0.4088 - val_mean_squared_error: 0.4088
Epoch 13/100
11610/11610 [==============================] - 1s 72us/sample - loss: 0.3923 - mean_squared_error: 0.3923 - val_loss: 0.4127 - val_mean_squared_error: 0.4127
Epoch 14/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3895 - mean_squared_error: 0.3895 - val_loss: 0.4019 - val_mean_squared_error: 0.4019
Epoch 15/100
11610/11610 [==============================] - 1s 59us/sample - loss: 0.3863 - mean_squared_error: 0.3863 - val_loss: 0.4008 - val_mean_squared_error: 0.4008
Epoch 16/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3835 - mean_squared_error: 0.3835 - val_loss: 0.3949 - val_mean_squared_error: 0.3949
Epoch 17/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3819 - mean_squared_error: 0.3819 - val_loss: 0.3942 - val_mean_squared_error: 0.3942
Epoch 18/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3813 - mean_squared_error: 0.3813 - val_loss: 0.3909 - val_mean_squared_error: 0.3909
Epoch 19/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3784 - mean_squared_error: 0.3784 - val_loss: 0.3882 - val_mean_squared_error: 0.3882
Epoch 20/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3763 - mean_squared_error: 0.3763 - val_loss: 0.3960 - val_mean_squared_error: 0.3960
Epoch 21/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3760 - mean_squared_error: 0.3760 - val_loss: 0.3876 - val_mean_squared_error: 0.3876
Epoch 22/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3730 - mean_squared_error: 0.3730 - val_loss: 0.3885 - val_mean_squared_error: 0.3885
Epoch 23/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3739 - mean_squared_error: 0.3739 - val_loss: 0.3854 - val_mean_squared_error: 0.3854
Epoch 24/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3701 - mean_squared_error: 0.3701 - val_loss: 0.3812 - val_mean_squared_error: 0.3812
Epoch 25/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3697 - mean_squared_error: 0.3697 - val_loss: 0.3796 - val_mean_squared_error: 0.3796
Epoch 26/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3701 - mean_squared_error: 0.3701 - val_loss: 0.3787 - val_mean_squared_error: 0.3787
Epoch 27/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3670 - mean_squared_error: 0.3670 - val_loss: 0.3777 - val_mean_squared_error: 0.3777
Epoch 28/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3661 - mean_squared_error: 0.3661 - val_loss: 0.3761 - val_mean_squared_error: 0.3761
Epoch 29/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3638 - mean_squared_error: 0.3638 - val_loss: 0.3738 - val_mean_squared_error: 0.3738
Epoch 30/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3677 - mean_squared_error: 0.3677 - val_loss: 0.3726 - val_mean_squared_error: 0.3726
Epoch 31/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3637 - mean_squared_error: 0.3637 - val_loss: 0.3703 - val_mean_squared_error: 0.3703
Epoch 32/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3651 - mean_squared_error: 0.3651 - val_loss: 0.3750 - val_mean_squared_error: 0.3750
Epoch 33/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3627 - mean_squared_error: 0.3627 - val_loss: 0.3714 - val_mean_squared_error: 0.3714
Epoch 34/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3593 - mean_squared_error: 0.3593 - val_loss: 0.3701 - val_mean_squared_error: 0.3701
Epoch 35/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3582 - mean_squared_error: 0.3582 - val_loss: 0.3681 - val_mean_squared_error: 0.3681
Epoch 36/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3622 - mean_squared_error: 0.3622 - val_loss: 0.3668 - val_mean_squared_error: 0.3668
Epoch 37/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3556 - mean_squared_error: 0.3556 - val_loss: 0.3789 - val_mean_squared_error: 0.3789
Epoch 38/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3618 - mean_squared_error: 0.3618 - val_loss: 0.3651 - val_mean_squared_error: 0.3651
Epoch 39/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.3546 - mean_squared_error: 0.3546 - val_loss: 0.3630 - val_mean_squared_error: 0.3630
Epoch 40/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3533 - mean_squared_error: 0.3533 - val_loss: 0.3897 - val_mean_squared_error: 0.3897
Epoch 41/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3589 - mean_squared_error: 0.3589 - val_loss: 0.3621 - val_mean_squared_error: 0.3621
Epoch 42/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.3540 - mean_squared_error: 0.3540 - val_loss: 0.3648 - val_mean_squared_error: 0.3648
Epoch 43/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.3554 - mean_squared_error: 0.3554 - val_loss: 0.3638 - val_mean_squared_error: 0.3638
Epoch 44/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3512 - mean_squared_error: 0.3512 - val_loss: 0.3575 - val_mean_squared_error: 0.3575
Epoch 45/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3549 - mean_squared_error: 0.3549 - val_loss: 0.3566 - val_mean_squared_error: 0.3566
Epoch 46/100
11610/11610 [==============================] - 1s 79us/sample - loss: 0.3497 - mean_squared_error: 0.3497 - val_loss: 0.3561 - val_mean_squared_error: 0.3561
Epoch 47/100
11610/11610 [==============================] - 1s 86us/sample - loss: 0.3470 - mean_squared_error: 0.3470 - val_loss: 0.3584 - val_mean_squared_error: 0.3584
Epoch 48/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3470 - mean_squared_error: 0.3470 - val_loss: 0.3526 - val_mean_squared_error: 0.3526
Epoch 49/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3469 - mean_squared_error: 0.3469 - val_loss: 0.3529 - val_mean_squared_error: 0.3529
Epoch 50/100
11610/11610 [==============================] - 1s 60us/sample - loss: 0.3492 - mean_squared_error: 0.3492 - val_loss: 0.3531 - val_mean_squared_error: 0.3531
Epoch 51/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3428 - mean_squared_error: 0.3428 - val_loss: 0.3601 - val_mean_squared_error: 0.3601
Epoch 52/100
11610/11610 [==============================] - 1s 73us/sample - loss: 0.3448 - mean_squared_error: 0.3448 - val_loss: 0.3517 - val_mean_squared_error: 0.3517
Epoch 53/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.3423 - mean_squared_error: 0.3423 - val_loss: 0.3618 - val_mean_squared_error: 0.3618
5160/1 [===================================================] - 0s 23us/sample - loss: 0.4712 - mean_squared_error: 0.3614
[0.36136142297308577, 0.3613615]

在這裡，我們可以看到使用子類的方式來定義層次是比較繁瑣和複雜的，如果我們只是想增加一個啟用函式層，這樣寫就顯得比較累贅了，所以我們可以使用lambda表示式來完成這個功能

import tensorflow as tf
from tensorflow.keras import layers, models
import numpy as np

if __name__ == "__main__":
    # 新增啟用函式層sotgplus:log(1+e^x)
    customized_softplus = layers.Lambda(lambda x: tf.nn.softplus(x))
    print(customized_softplus([-10., -5., 0., 5., 10.]))
    model = models.Sequential([
        layers.Dense(30, activation='relu', input_shape=[None, 5]),
        layers.Dense(1),
        customized_softplus
        # 它等同於下面任意一行程式碼
        # layers.Dense(1, activation='softplus'),
        # layers.Dense(1), layers.Activation('softplus')
    ])
    print(model.summary())

執行結果

tf.Tensor([4.5417706e-05 6.7153489e-03 6.9314718e-01 5.0067153e+00 1.0000046e+01], shape=(5,), dtype=float32)
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, None, 30)          180       
_________________________________________________________________
dense_1 (Dense)              (None, None, 1)           31        
_________________________________________________________________
lambda (Lambda)              (None, None, 1)           0         
=================================================================
Total params: 211
Trainable params: 211
Non-trainable params: 0
_________________________________________________________________
None

tf.function函式轉換圖結構

這裡我們有一個概念，那就是tensorflow的型別(常量，變數)以及它的函式都是被優化過，更加適合於使用GPU、TPU進行加速，那麼對於普通的Python庫函式是沒有這個功能的。那麼對於普通的Python庫函式能否使用tensorflow的這種優化呢，答案是肯定的。

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import timeit
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    def scaled_elu(z, scale=1.0, alpha=1.0):
        # z >= 0 ? scale * z : scale * alpha * tf.nn.elu(z)
        is_positive = tf.greater_equal(z, 0.0)
        return scale * tf.where(is_positive, z, alpha * tf.nn.elu(z))

    t1 = tf.constant(-3.)
    t2 = tf.constant([-3., -2.5])
    print(scaled_elu(t1))
    print(scaled_elu(t2))
    scaled_elu_tf = tf.function(scaled_elu)
    print(scaled_elu_tf(t1))
    print(scaled_elu_tf(t2))
    print(scaled_elu_tf.python_function is scaled_elu)
    %timeit scaled_elu(tf.random.normal((1000, 1000)))
    %timeit scaled_elu_tf(tf.random.normal((1000, 1000)))

執行結果

tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
True
10 loops, best of 5: 25.8 ms per loop
10 loops, best of 5: 23.7 ms per loop

這裡我們可以把scaled_elu看成是一個普通的Python函式，當我們對這個函式進行了圖模型化scaled_elu_tf = tf.function(scaled_elu)後，當我們對其進行大型矩陣計算中，我們可以看到圖模型化後的scaled_elu_tf比之前scaled_elu快了一點點。不過這裡必須使用TPU環境才有效果，否則是比較慢的。除了這種直接轉的方式，我們還可以通過打標籤的方式

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import timeit
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    def scaled_elu(z, scale=1.0, alpha=1.0):
        # z >= 0 ? scale * z : scale * alpha * tf.nn.elu(z)
        is_positive = tf.greater_equal(z, 0.0)
        return scale * tf.where(is_positive, z, alpha * tf.nn.elu(z))

    t1 = tf.constant(-3.)
    t2 = tf.constant([-3., -2.5])
    print(scaled_elu(t1))
    print(scaled_elu(t2))
    scaled_elu_tf = tf.function(scaled_elu)
    print(scaled_elu_tf(t1))
    print(scaled_elu_tf(t2))
    print(scaled_elu_tf.python_function is scaled_elu)
    %timeit scaled_elu(tf.random.normal((1000, 1000)))
    %timeit scaled_elu_tf(tf.random.normal((1000, 1000)))

    @tf.function
    def converge_to_2(n_iters):
        # 1 + 1/2 + 1/2^2 + ... + 1/2^n
        total = tf.constant(0.)
        increment = tf.constant(1.)
        for _ in range(n_iters):
            total += increment
            increment /= 2.0
        return total

    print(converge_to_2(20))

執行結果

tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
True
10 loops, best of 5: 26 ms per loop
10 loops, best of 5: 24 ms per loop
tf.Tensor(1.9999981, shape=(), dtype=float32)

我們可以來看一下這兩個函式轉成圖結構後的程式碼

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import timeit
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from IPython.display import display, Markdown

if __name__ == "__main__":

    def scaled_elu(z, scale=1.0, alpha=1.0):
        # z >= 0 ? scale * z : scale * alpha * tf.nn.elu(z)
        is_positive = tf.greater_equal(z, 0.0)
        return scale * tf.where(is_positive, z, alpha * tf.nn.elu(z))

    t1 = tf.constant(-3.)
    t2 = tf.constant([-3., -2.5])
    print(scaled_elu(t1))
    print(scaled_elu(t2))
    scaled_elu_tf = tf.function(scaled_elu)
    print(scaled_elu_tf(t1))
    print(scaled_elu_tf(t2))
    print(scaled_elu_tf.python_function is scaled_elu)
    %timeit scaled_elu(tf.random.normal((1000, 1000)))
    %timeit scaled_elu_tf(tf.random.normal((1000, 1000)))

    # @tf.function
    def converge_to_2(n_iters):
        # 1 + 1/2 + 1/2^2 + ... + 1/2^n
        total = tf.constant(0.)
        increment = tf.constant(1.)
        for _ in range(n_iters):
            total += increment
            increment /= 2.0
        return total

    print(converge_to_2(20))

    def display_tf_code(func):
        code = tf.autograph.to_code(func)
        display(Markdown('```python\n{}\n```'.format(code)))

    display_tf_code(scaled_elu)
    display_tf_code(converge_to_2)

執行結果

tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
True
10 loops, best of 5: 25.7 ms per loop
10 loops, best of 5: 23.9 ms per loop
tf.Tensor(1.9999981, shape=(), dtype=float32)
def tf__scaled_elu(z, scale=None, alpha=None):
    with ag__.FunctionScope('scaled_elu', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:
        do_return = False
        retval_ = ag__.UndefinedReturnValue()
        is_positive = ag__.converted_call(ag__.ld(tf).greater_equal, (ag__.ld(z), 0.0), None, fscope)
        try:
            do_return = True
            retval_ = (ag__.ld(scale) * ag__.converted_call(ag__.ld(tf).where, (ag__.ld(is_positive), ag__.ld(z), (ag__.ld(alpha) * ag__.converted_call(ag__.ld(tf).nn.elu, (ag__.ld(z),), None, fscope))), None, fscope))
        except:
            do_return = False
            raise
        return fscope.ret(retval_, do_return)
def tf__converge_to(n_iters):
    with ag__.FunctionScope('converge_to_2', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:
        do_return = False
        retval_ = ag__.UndefinedReturnValue()
        total = ag__.converted_call(ag__.ld(tf).constant, (0.0,), None, fscope)
        increment = ag__.converted_call(ag__.ld(tf).constant, (1.0,), None, fscope)

        def get_state():
            return (total, increment)

        def set_state(vars_):
            nonlocal increment, total
            (total, increment) = vars_

        def loop_body(itr):
            nonlocal increment, total
            _ = itr
            total = ag__.ld(total)
            total += increment
            increment = ag__.ld(increment)
            increment /= 2.0
        _ = ag__.Undefined('_')
        ag__.for_stmt(ag__.converted_call(ag__.ld(range), (ag__.ld(n_iters),), None, fscope), None, loop_body, get_state, set_state, ('total', 'increment'), {'iterate_names': '_'})
        try:
            do_return = True
            retval_ = ag__.ld(total)
        except:
            do_return = False
            raise
        return fscope.ret(retval_, do_return)

之前我們使用的都是常量，現在我們來看看對於變數來說，是什麼情況

def add_21():
        var = tf.Variable(0.)
        return var.assign_add(21) # +=

print(add_21())

執行結果

<tf.Variable 'UnreadVariable' shape=() dtype=float32, numpy=21.0>

現在我們給這段程式碼加上標籤

@tf.function
def add_21():
        var = tf.Variable(0.)
        return var.assign_add(21) # +=

print(add_21())

執行結果

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-8-806807c51c7b> in <module>()
      4         return var.assign_add(21) # +=
      5 
----> 6 print(add_21())

7 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
    992           except Exception as e:  # pylint:disable=broad-except
    993             if hasattr(e, "ag_error_metadata"):
--> 994               raise e.ag_error_metadata.to_exception(e)
    995             else:
    996               raise

ValueError: in user code:

    <ipython-input-8-806807c51c7b>:3 add_21  *
        var = tf.Variable(0.)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py:268 __call__  **
        return cls._variable_v2_call(*args, **kwargs)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py:262 _variable_v2_call
        shape=shape)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py:67 getter
        return captured_getter(captured_previous, **kwargs)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py:765 invalid_creator_scope
        "tf.function-decorated function tried to create "

    ValueError: tf.function-decorated function tried to create variables on non-first call.

它會直接報錯，這個主要是對於tensorflow的圖結構來說，它是不能直接在函式內部定義變數的，所以我們需要將變數定義到函式外部。

var = tf.Variable(0.)

@tf.function
def add_21():
        return var.assign_add(21) # +=

print(add_21())

執行結果

tf.Tensor(21.0, shape=(), dtype=float32)

由於Python是一個弱型別的語言，有的時候它可以輸入多種不同的型別，現在我們要給圖結構函式限制輸入的型別

@tf.function(input_signature=[tf.TensorSpec([None], tf.int32, name='x')])
def cube(z):
    return tf.pow(z, 3)

print(cube(tf.constant([1., 2., 3.])))
print(cube(tf.constant([1, 2, 3])))

執行結果

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-10-53f91d549b69> in <module>()
      3     return tf.pow(z, 3)
      4 
----> 5 print(cube(tf.constant([1., 2., 3.])))
      6 print(cube(tf.constant([1, 2, 3])))

3 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py in _convert_inputs_to_signature(inputs, input_signature, flat_input_signature)
   2890       flatten_inputs)):
   2891     raise ValueError("Python inputs incompatible with input_signature:\n%s" %
-> 2892                      format_error_message(inputs, input_signature))
   2893 
   2894   if need_packing:

ValueError: Python inputs incompatible with input_signature:
  inputs: (
    tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32))
  input_signature: (
    TensorSpec(shape=(None,), dtype=tf.int32, name='x'))

這裡我們給輸入的引數限定為int32的整型，當我們輸入整型的時候

@tf.function(input_signature=[tf.TensorSpec([None], tf.int32, name='x')])
def cube(z):
    return tf.pow(z, 3)

# print(cube(tf.constant([1., 2., 3.])))
print(cube(tf.constant([1, 2, 3])))

執行結果

tf.Tensor([ 1  8 27], shape=(3,), dtype=int32)

現在我們來看一下這個圖的一些資訊

# get_concrete_function -> 新增輸入簽名 ->儲存模型
cube_func_int32 = cube.get_concrete_function(tf.TensorSpec([None], tf.int32))
print(cube_func_int32)
print(cube_func_int32 is cube.get_concrete_function(tf.TensorSpec([5], tf.int32)))
print(cube_func_int32 is cube.get_concrete_function(tf.constant([1, 2, 3])))

執行結果

ConcreteFunction cube(z)
  Args:
    z: int32 Tensor, shape=(None,)
  Returns:
    int32 Tensor, shape=(None,)
True
True

在cube_func_int32物件中就有圖的資訊

print(cube_func_int32.graph)
# 檢視圖中的操作
print(cube_func_int32.graph.get_operations())
# 檢視立方操作
pow_op = cube_func_int32.graph.get_operations()[2]
print(pow_op)
# 檢視該操作的輸入和輸出
print(list(pow_op.inputs))
print(list(pow_op.outputs))
# 通過操作名字獲取操作
print(cube_func_int32.graph.get_operation_by_name("x"))
# 通過張量名字獲取操作
print(cube_func_int32.graph.get_tensor_by_name("x:0"))
# 列印圖定義
print(cube_func_int32.graph.as_graph_def())

執行結果

FuncGraph(name=cube, id=140354824183312)
[<tf.Operation 'x' type=Placeholder>, <tf.Operation 'Pow/y' type=Const>, <tf.Operation 'Pow' type=Pow>, <tf.Operation 'Identity' type=Identity>]
name: "Pow"
op: "Pow"
input: "x"
input: "Pow/y"
attr {
  key: "T"
  value {
    type: DT_INT32
  }
}

[<tf.Tensor 'x:0' shape=(None,) dtype=int32>, <tf.Tensor 'Pow/y:0' shape=() dtype=int32>]
[<tf.Tensor 'Pow:0' shape=(None,) dtype=int32>]
name: "x"
op: "Placeholder"
attr {
  key: "_user_specified_name"
  value {
    s: "x"
  }
}
attr {
  key: "dtype"
  value {
    type: DT_INT32
  }
}
attr {
  key: "shape"
  value {
    shape {
      dim {
        size: -1
      }
    }
  }
}

Tensor("x:0", shape=(None,), dtype=int32)
node {
  name: "x"
  op: "Placeholder"
  attr {
    key: "_user_specified_name"
    value {
      s: "x"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: -1
        }
      }
    }
  }
}
node {
  name: "Pow/y"
  op: "Const"
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 3
      }
    }
  }
}
node {
  name: "Pow"
  op: "Pow"
  input: "x"
  input: "Pow/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "Identity"
  op: "Identity"
  input: "Pow"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
}
versions {
  producer: 808
}

我們這裡可以看到該圖中有4個操作，這裡有Placeholder的概念，Placeholder是tensorflow1中經常用到的圖，可以顯示定義，在2.0中就不需要了，它是用來放輸入的，必須要放進去一個輸入才能得到一個輸出。那麼圖資訊對我們有什麼用呢？一般會用在兩個地方，第一個地方就是如何去儲存模型；第二個地方就是一旦儲存了模型之後，如何把一個模型給載入進來，然後我們載入模型去做infos的時候會經常用到上面這些函式。

自定義求導

我們來看一個近似求導的方法

if __name__ == "__main__":

    def f(x):
        return 3. * x**2 + 2 * x -1

    def approximate_derivative(f, x, eps=1e-3):
        # 近似求導
        return (f(x + eps) - f(x - eps)) / (2. * eps)

    print(approximate_derivative(f, 1))

    def g(x1, x2):
        return (x1 + 5) * x2**2

    def approximate_gradient(g, x1, x2, eps=1e-3):
        # x1的偏導
        dg_x1 = approximate_derivative(lambda x: g(x, x2), x1, eps)
        # x2的偏導
        dg_x2 = approximate_derivative(lambda x: g(x1, x), x2, eps)
        return dg_x1, dg_x2

    print(approximate_gradient(g, 2., 3.))

執行結果

7.999999999999119
(8.999999999993236, 41.999999999994486)

對於3x^2+2x-1這個式子的導數，我們知道為3*2x+2，當x=1的時候，結果為8，我們使用近似求導的方法得到了7.999。對於第二個式子的x1的偏導數為x2^2，當x2=3的時候，結果為9，x2的偏導數為2(x1+5)x2，當x1=2,x2=3的時候，結果為42。現在我們來看看在tensorflow中如何求導數的

import tensorflow as tf

if __name__ == "__main__":

    def f(x):
        return 3. * x**2 + 2 * x -1

    def approximate_derivative(f, x, eps=1e-3):
        # 近似求導
        return (f(x + eps) - f(x - eps)) / (2. * eps)

    print(approximate_derivative(f, 1))

    def g(x1, x2):
        return (x1 + 5) * x2**2

    def approximate_gradient(g, x1, x2, eps=1e-3):
        # x1的偏導
        dg_x1 = approximate_derivative(lambda x: g(x, x2), x1, eps)
        # x2的偏導
        dg_x2 = approximate_derivative(lambda x: g(x1, x), x2, eps)
        return dg_x1, dg_x2

    print(approximate_gradient(g, 2., 3.))

    x1 = tf.Variable(2.)
    x2 = tf.Variable(3.)
    # 此處如果不加persistent=True，則tape只能使用一次
    # 加了persistent=True必須手動釋放資源
    with tf.GradientTape(persistent=True) as tape:
        z = g(x1, x2)
    dz_x1 = tape.gradient(z, x1)
    dz_x2 = tape.gradient(z, x2)
    print(dz_x1, dz_x2)
    del tape
    
    with tf.GradientTape() as tape:
        z = g(x1, x2)
    # 結果為一個元組
    dz_x1x2 = tape.gradient(z, [x1, x2])
    print(dz_x1x2)

執行結果

7.999999999999119
(8.999999999993236, 41.999999999994486)
tf.Tensor(9.0, shape=(), dtype=float32) tf.Tensor(42.0, shape=(), dtype=float32)
[<tf.Tensor: id=46, shape=(), dtype=float32, numpy=9.0>, <tf.Tensor: id=52, shape=(), dtype=float32, numpy=42.0>]

以上我嗎使用的是變數，但是對於常量來說，tape是無法直接計算出它的導數值的，我們需要進行一個變動，需要關注這兩個常量。

x1 = tf.constant(2.)
x2 = tf.constant(3.)
with tf.GradientTape() as tape:
    tape.watch(x1)
    tape.watch(x2)
    z = g(x1, x2)
# 結果為一個元組
dz_x1x2 = tape.gradient(z, [x1, x2])
print(dz_x1x2)

執行結果

[<tf.Tensor: id=61, shape=(), dtype=float32, numpy=9.0>, <tf.Tensor: id=67, shape=(), dtype=float32, numpy=42.0>]

我們也可以對兩個函式連立求一個變數的導數

x = tf.Variable(5.0)
with tf.GradientTape() as tape:
    z1 = 3 * x
    z2 = x**2
print(tape.gradient([z1, z2], x))

執行結果

tf.Tensor(13.0, shape=(), dtype=float32)

這個13是怎麼來對呢？首先z1'=3，z2'=2x，當x=5的時候，兩個函式連立，有3+2*5=13。現在我們來看一下求二階導數

x1 = tf.Variable(2.)
x2 = tf.Variable(3.)
with tf.GradientTape(persistent=True) as out_tape:
    with tf.GradientTape(persistent=True) as inner_tape:
        z = g(x1, x2)
    inner_grads = inner_tape.gradient(z, [x1, x2])
out_grads = [out_tape.gradient(inner_grad, [x1, x2]) for inner_grad in inner_grads]
print(out_grads)
del inner_tape
del out_tape

執行結果

[[None, <tf.Tensor: id=127, shape=(), dtype=float32, numpy=6.0>], [<tf.Tensor: id=138, shape=(), dtype=float32, numpy=6.0>, <tf.Tensor: id=136, shape=(), dtype=float32, numpy=14.0>]]

這裡有關二階偏導數的內容可以參考高等數學整理(二)

梯度下降

learning_rate = 0.1
x = tf.Variable(0.)
for _ in range(100):
    with tf.GradientTape() as tape:
        z = f(x)
    dz_dx = tape.gradient(z, x)
    x.assign_sub(learning_rate * dz_dx)
print(x)

執行結果

<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-0.3333333>

這個是一個最簡單的梯度下降演算法，tensorflow裡面有一個專門用來做梯度下降的優化器

from tensorflow.keras import optimizers

learning_rate = 0.1
x = tf.Variable(0.)
# 使用隨機梯度下降
optimizer = optimizers.SGD(learning_rate=learning_rate)
for _ in range(100):
    with tf.GradientTape() as tape:
        z = f(x)
    dz_dx = tape.gradient(z, x)
    optimizer.apply_gradients([(dz_dx, x)])
print(x)

執行結果

<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-0.3333333>

現在我們來看一下如何使用這種自定義求導的方式來對加利福尼亞的房價進行迴歸操作

「其他文章」