Python tensorflow深度學習中的過擬合防治：早停策略,動態調整學習率,Dropout,正則化調整loss

輸入層：
 x1    x2    x3    x4
  |     |     |     |
 [w11] [w12] [w13] [w14]  <- 對應神經元 1 的權重
  |     |     |     |
 [w21] [w22] [w23] [w24]  <- 對應神經元 2 的權重
  |     |     |     |
 [w31] [w32] [w33] [w34]  <- 對應神經元 3 的權重
  |     |     |     |
偏置： b1    b2    b3      <- 偏置項，只與輸出神經元數量一致

import tensorflow as tf

# 定義模型
model = tf.keras.Sequential([
    tf.keras.layers.Dense(64, activation='relu', 
    kernel_regularizer=tf.keras.regularizers.l2(0.01), bias_regularizer=tf.keras.regularizers.l2(0.01)),
    tf.keras.layers.Dense(1, activation='sigmoid')
])

# 編譯模型
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])

from tensorflow.keras.regularizers import l1

# 使用 L1 正則化
model = tf.keras.Sequential([
    tf.keras.layers.Dense(64, activation='relu', 
    kernel_regularizer=l1(0.01))
])

from tensorflow.keras.regularizers import l2

# 使用 L2 正則化
model = tf.keras.Sequential([
    tf.keras.layers.Dense(64, activation='relu', 
    kernel_regularizer=l2(0.01))
])

from tensorflow.keras.regularizers import l1_l2

# 使用 L1 和 L2 結合的正則化
model = tf.keras.Sequential([
    tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01))
])

from tensorflow.keras.callbacks import EarlyStopping

early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

history = model.fit(
    train_data, train_labels,
    validation_data=(val_data, val_labels),
    epochs=100,
    callbacks=[early_stopping]
)

model = tf.keras.Sequential([
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Dropout(0.5),  # 50% 的神經元被隨機屏蔽
    tf.keras.layers.Dense(64, activation='relu'),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(1, activation='sigmoid')
])

from tensorflow.keras.callbacks import ReduceLROnPlateau
#ReduceLROnPlateau = 
#Reduce LR(Learning Rate) On Plateau(停滯)

reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
factor=0.5, patience=3, min_lr=1e-6)

history = model.fit(
    train_data, train_labels,
    validation_data=(val_data, val_labels),
    epochs=100,
    callbacks=[reduce_lr]
)

from tensorflow.keras.callbacks import LearningRateScheduler

def scheduler(epoch, lr):
    if epoch < 10:
        return lr
    else:
        return lr * tf.math.exp(-0.1)

lr_scheduler = LearningRateScheduler(scheduler)

history = model.fit(
    train_data, train_labels,
    validation_data=(val_data, val_labels),
    epochs=50,
    callbacks=[lr_scheduler] 
    #callbacks 可以同時使用 早停 與 動態調整學習率
)

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
def create_model(input_dim):
    """
    input_dim: 輸入特徵的維度
    """
    model = Sequential([
        Dense(64, activation='relu', input_dim=input_dim),#input_shape=(16,)

        Dropout(0.2),
        #加入正則化 kernel_regularizer bias_regularizer
        Dense(32, activation='relu', 
              kernel_regularizer=tf.keras.regularizers.l2(0.001), 
              bias_regularizer=tf.keras.regularizers.l2(0.001)),
        Dropout(0.2),
        Dense(16, activation='relu', 
              kernel_regularizer=tf.keras.regularizers.l2(0.001), 
              bias_regularizer=tf.keras.regularizers.l2(0.001)),

        Dense(1)  # 回歸問題，輸出層不使用激活函數
    ])
    
    model.compile(
        optimizer=Adam(learning_rate=0.001),
        loss='mse',
        metrics=['mae'] #['mae', 'mse']
    )
    return model

model = create_model(input_dim)
# 訓練模型
history = model.fit(
                X_train_scaled, y_train_scaled,
                epochs=100,
                batch_size=32,
                validation_split=0.2,
                # 自動從訓練數據中取20%作為驗證集,torch需手動實現
                verbose=0,
                callbacks=[early_stopping, reduceLR] 
                #early_stopping, reduceLR 前面需要定義
            )