{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "a93201f6",
      "metadata": {
        "id": "a93201f6"
      },
      "source": [
        "# **自定義訓練流程 (Pytorch)**\n",
        "此份程式碼為 Custom_training 的 PyTorch 參考寫法。\n",
        "## 本章節內容大綱\n",
        "* ### [建立資料集](#CreateDataset)\n",
        "* ### [建構模型](#BuildModel)\n",
        "* ### [訓練模型](#TrainModel)\n",
        "* ### [評估模型](#EvaluateModel)\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "40c02e63",
      "metadata": {
        "id": "40c02e63"
      },
      "source": [
        "## 匯入套件"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9400e44f",
      "metadata": {
        "id": "9400e44f"
      },
      "outputs": [],
      "source": [
        "!pip install torchsummary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f0753308",
      "metadata": {
        "id": "f0753308"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# PyTorch 相關套件\n",
        "import torch\n",
        "from torchsummary import summary"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "159e7a52",
      "metadata": {
        "id": "159e7a52"
      },
      "source": [
        "<a name=\"CreateDataset\"></a>\n",
        "## 建立資料集"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4b9a560c",
      "metadata": {
        "id": "4b9a560c"
      },
      "outputs": [],
      "source": [
        "# 上傳資料\n",
        "!wget -q https://github.com/TA-aiacademy/course_3.0/releases/download/DL/Data_part4.zip\n",
        "!unzip -q Data_part4.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "022a9444",
      "metadata": {
        "id": "022a9444"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv('./Data/bodyperformance.csv')\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "06eb5205",
      "metadata": {
        "id": "06eb5205"
      },
      "outputs": [],
      "source": [
        "X = torch.from_numpy(df.iloc[:, :-1].values)\n",
        "y = torch.from_numpy(df['class'].values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4a6a8b48",
      "metadata": {
        "id": "4a6a8b48"
      },
      "outputs": [],
      "source": [
        "y_onehot = torch.nn.functional.one_hot(y).double()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "df21eac9",
      "metadata": {
        "id": "df21eac9"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(X, y_onehot,\n",
        "                                                      test_size=0.2,\n",
        "                                                      random_state=17,\n",
        "                                                      stratify=y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4235fc7d",
      "metadata": {
        "id": "4235fc7d"
      },
      "outputs": [],
      "source": [
        "print(f'X_train shape: {X_train.shape}')\n",
        "print(f'X_valid shape: {X_valid.shape}')\n",
        "print(f'y_train shape: {y_train.shape}')\n",
        "print(f'y_valid shape: {y_valid.shape}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b4292301",
      "metadata": {
        "id": "b4292301"
      },
      "outputs": [],
      "source": [
        "# Feature scaling\n",
        "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
        "sc = StandardScaler()\n",
        "X_train = sc.fit_transform(X_train, y_train)\n",
        "X_valid = sc.transform(X_valid)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "be65be82",
      "metadata": {
        "id": "be65be82"
      },
      "outputs": [],
      "source": [
        "batch_size = 64\n",
        "\n",
        "# 準備訓練資料集\n",
        "train_dataset = torch.utils.data.TensorDataset(torch.Tensor(X_train),\n",
        "                                               torch.Tensor(y_train))\n",
        "train_dataset = torch.utils.data.DataLoader(train_dataset,\n",
        "                                            batch_size=batch_size,\n",
        "                                            shuffle=True)\n",
        "\n",
        "# 準備驗證資料集\n",
        "val_dataset = torch.utils.data.TensorDataset(torch.Tensor(X_valid),\n",
        "                                             torch.Tensor(y_valid))\n",
        "val_dataset = torch.utils.data.DataLoader(val_dataset,\n",
        "                                          batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "bc77973b",
      "metadata": {
        "id": "bc77973b"
      },
      "source": [
        "<a name=\"BuildModel\"></a>\n",
        "## 建構模型"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e1db7a5f",
      "metadata": {
        "id": "e1db7a5f"
      },
      "outputs": [],
      "source": [
        "def build_model(input_shape, output_shape):\n",
        "    torch.cuda.empty_cache()\n",
        "    torch.manual_seed(17)\n",
        "\n",
        "    model = torch.nn.Sequential(\n",
        "        torch.nn.Linear(input_shape, 32),\n",
        "        torch.nn.SiLU(),\n",
        "        torch.nn.Linear(32, 32),\n",
        "        torch.nn.SiLU(),\n",
        "        torch.nn.Linear(32, 32),\n",
        "        torch.nn.SiLU(),\n",
        "        torch.nn.Linear(32, output_shape))\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "17d78b2d",
      "metadata": {
        "id": "17d78b2d"
      },
      "outputs": [],
      "source": [
        "model = build_model(X_train[0].shape[0], y_onehot.shape[1])\n",
        "summary(model, X_train[0].shape, device='cpu')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3e9d5200",
      "metadata": {
        "id": "3e9d5200"
      },
      "outputs": [],
      "source": [
        "# Instantiate an optimizer to train the model\n",
        "loss_fn = torch.nn.CrossEntropyLoss()\n",
        "# Instantiate a loss function\n",
        "optimizer = torch.optim.NAdam(model.parameters(), lr=0.001)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "d3befb96",
      "metadata": {
        "id": "d3befb96"
      },
      "source": [
        "<a name=\"TrainModel\"></a>\n",
        "## 訓練模型"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d86f336c",
      "metadata": {
        "id": "d86f336c"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import tqdm\n",
        "\n",
        "# 將模型和損失函數放入 GPU 記憶體當中\n",
        "if torch.cuda.is_available():\n",
        "    model.cuda()\n",
        "    loss_fn.cuda()\n",
        "\n",
        "epochs = 10\n",
        "\n",
        "# 創建 list 分別存放訓練集 acc, loss 和驗證集 acc\n",
        "train_acc_list, train_loss_list = [], []\n",
        "val_acc_list, val_loss_list = [], []\n",
        "\n",
        "# 訓練的迭代過程\n",
        "for epoch in range(epochs):\n",
        "    start_time = time.time()\n",
        "    t_bar = tqdm.tqdm_notebook(enumerate(train_dataset),\n",
        "                               total=len(train_dataset),\n",
        "                               desc=f'Epoch {epoch}')\n",
        "    train_correct = 0\n",
        "    train_samples = 0\n",
        "    for step, (x_batch_train, y_batch_train) in t_bar:\n",
        "        if torch.cuda.is_available():\n",
        "            x_batch_train = x_batch_train.cuda()\n",
        "            y_batch_train = y_batch_train.cuda()\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(x_batch_train)\n",
        "        loss_value = loss_fn(outputs, y_batch_train)\n",
        "        loss_value.backward()  # 計算參數上的梯度\n",
        "        optimizer.step()  # 更新參數\n",
        "\n",
        "        predict_cls = torch.argmax(outputs, 1)\n",
        "        target_cls = torch.argmax(y_batch_train, 1)\n",
        "        train_samples += target_cls.size(0)\n",
        "        train_correct += (predict_cls == target_cls).sum().item()\n",
        "\n",
        "    print('Training loss over epoch: %.4f' % (float(loss_value.item()),))\n",
        "    train_acc = train_correct / train_samples  # 平均所有批次的評估結果\n",
        "    print('Training acc over epoch: %.4f' % (float(train_acc),))\n",
        "\n",
        "    # 將訓練的評估結果儲存下來\n",
        "    train_acc_list.append(train_acc)\n",
        "    train_loss_list.append(loss_value.cpu().detach().numpy())\n",
        "\n",
        "    # 驗證集的迭代結果\n",
        "    val_correct = 0\n",
        "    val_samples = 0\n",
        "    with torch.no_grad():\n",
        "        for x_batch_val, y_batch_val in val_dataset:\n",
        "            if torch.cuda.is_available():\n",
        "                x_batch_val = x_batch_val.cuda()\n",
        "                y_batch_val = y_batch_val.cuda()\n",
        "            val_logits = model(x_batch_val)\n",
        "            val_pred_cls = torch.argmax(val_logits, 1)\n",
        "            val_target_cls = torch.argmax(y_batch_val, 1)\n",
        "            val_samples += val_target_cls.size(0)\n",
        "            val_correct += (val_pred_cls == val_target_cls).sum().item()\n",
        "\n",
        "    val_loss = loss_fn(y_batch_val, val_logits)  # 計算最後批次的損失值\n",
        "\n",
        "    # 印出每個迭代回合的驗證評估結果\n",
        "    print('Training loss over epoch: %.4f' % (float(loss_value.item()),))\n",
        "    val_acc = val_correct / val_samples  # 平均所有批次的評估結果\n",
        "    print('Validation acc: %.4f' % (float(val_acc),))\n",
        "\n",
        "    # 將驗證的評估結果儲存下來\n",
        "    val_acc_list.append(val_acc)\n",
        "    val_loss_list.append(val_loss.cpu().detach().numpy())\n",
        "\n",
        "    print('Time taken: %.2fs' % (time.time() - start_time))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7d2130ed",
      "metadata": {
        "id": "7d2130ed"
      },
      "source": [
        "<a name=\"EvaluateModel\"></a>\n",
        "## 評估模型"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c4a9c6a0",
      "metadata": {
        "id": "c4a9c6a0"
      },
      "source": [
        "* ### 視覺化訓練過程的評估指標 （Visualization）"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "51562452",
      "metadata": {
        "id": "51562452"
      },
      "outputs": [],
      "source": [
        "plt.figure(figsize=(15, 4))\n",
        "plt.subplot(1, 2, 1)\n",
        "plt.plot(range(len(train_loss_list)), train_loss_list, label='train_loss')\n",
        "plt.plot(range(len(val_loss_list)), val_loss_list, label='valid_loss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.plot(range(len(train_acc_list)), train_acc_list, label='train_acc')\n",
        "plt.plot(range(len(val_acc_list)), val_acc_list, label='valid_acc')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "51db7532",
      "metadata": {
        "id": "51db7532"
      },
      "source": [
        "* ### 模型預測（Model predictions）"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e8d61857",
      "metadata": {
        "id": "e8d61857"
      },
      "outputs": [],
      "source": [
        "val_pred = []\n",
        "for x_val, y_val in val_dataset:\n",
        "    if torch.cuda.is_available():\n",
        "        x_val = x_val.cuda()\n",
        "        y_val = y_val.cuda()\n",
        "    val_pred += list(model(x_val).argmax(-1).flatten().cpu())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "32d11924",
      "metadata": {
        "id": "32d11924"
      },
      "outputs": [],
      "source": [
        "val_pred[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a43f958d",
      "metadata": {
        "id": "a43f958d"
      },
      "outputs": [],
      "source": [
        "len(val_pred)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e2be1cee",
      "metadata": {
        "id": "e2be1cee"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import classification_report\n",
        "print(classification_report(y_valid.argmax(-1), val_pred))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1d018683",
      "metadata": {
        "id": "1d018683"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.12"
    },
    "colab": {
      "provenance": []
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}