{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"ConvLSTM.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyMXox37jjcWOTMtoI++NxCW"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"KChrGBiCjFJF","executionInfo":{"status":"ok","timestamp":1623580255557,"user_tz":-540,"elapsed":2437,"user":{"displayName":"‍이혜리[학생](소프트웨어융합대학 컴퓨터공학과)","photoUrl":"","userId":"16579190241035414660"}}},"source":["import numpy as np\n","import cv2\n","import matplotlib.pyplot as plt\n","import os\n","import random\n","from math import ceil\n","from tqdm import tqdm_notebook\n","import tensorflow as tf\n","from keras.models import Sequential\n","from keras.layers import Dense\n","from keras.layers import Flatten\n","from keras.layers import Dropout\n","from keras.layers import LSTM\n","from keras.layers import TimeDistributed\n","from keras.layers import ConvLSTM2D\n","from keras.utils.np_utils  import to_categorical\n","from keras.models import load_model"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"id":"pRoXlH-ejqaV","executionInfo":{"status":"ok","timestamp":1623580304615,"user_tz":-540,"elapsed":361,"user":{"displayName":"‍이혜리[학생](소프트웨어융합대학 컴퓨터공학과)","photoUrl":"","userId":"16579190241035414660"}}},"source":["BASE_PATH = './Capston2'\n","DATASET_PATH = BASE_PATH+ '/data'"],"execution_count":6,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"MW6_oaLWFhXH"},"source":["### Image sequence를 numpy로 만들기"]},{"cell_type":"markdown","metadata":{"id":"27d5FeyVFyD4"},"source":["1. 각 image sequence의 data directory path 가져오고 train, test dataset 나누기"]},{"cell_type":"code","metadata":{"id":"WCxaqiIAkiJ8"},"source":["data_dir_path_list = []\n","for top, dir, f in os.walk(DATASET_PATH):\n","    if len(dir) == 0:\n","        data_dir_path_list.append(top)\n","\n","random.shuffle(data_dir_path_list)\n","\n","tr_ratio = 0.8\n","point = ceil(len(data_dir_path_list) * tr_ratio)\n","train_data_dir_path = data_dir_path_list[:point]\n","test_data_dir_path = data_dir_path_list[point:]\n","\n","# train_data_dir_path_numpy = np.array(train_data_dir_path)\n","# test_data_dir_path_numpy = np.array(test_data_dir_path)\n","# np.save(BASE_PATH + '/train_dir_path', train_data_dir_path_numpy)\n","# np.save(BASE_PATH + '/test_dir_path', test_data_dir_path_numpy)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Q7IcRpY-98LU"},"source":["2. image sequence를 5D tensor로 만들어서 저장"]},{"cell_type":"code","metadata":{"id":"3inQO4yQnOfp"},"source":["def get_image_sequence(data_dir_path_list, typ):\n","    idx = {'fight':0 , 'swoon':1, 'normal':2}\n","    dataset = []\n","    label = []\n","    for base_path in tqdm_notebook(data_dir_path_list):\n","        tmp = []\n","        im_list = sorted(os.listdir(base_path))\n","        for i in im_list:\n","            path = os.path.join(base_path,i)\n","            img = cv2.imread(path)\n","            img = cv2.resize(img, dsize=(0, 0), fx=0.045, fy=0.045, interpolation=cv2.INTER_AREA)\n","            tmp.append(np.array(img)) \n","        if len(tmp) == 20:\n","            dataset.append(np.array(tmp))\n","            label.append(idx[base_path.split('/')[-3]])\n","\n","    dataset = np.array(dataset)\n","    label = to_categorical(label)\n","\n","    np.save('./drive/MyDrive/Capston2/{}_img_seq'.format(typ), dataset)\n","    np.save('./drive/MyDrive/Capston2/{}_label'.format(typ), label)\n","\n","get_image_sequence(train_data_dir_path, 'train')\n","get_image_sequence(test_data_dir_path, 'test')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"McUXDRXE-LHR"},"source":["### ConvLSTM model"]},{"cell_type":"code","metadata":{"id":"HWfLFz7g4cac"},"source":["# data load\n","train_img = np.load(BASE_PATH+'/train_img_seq.npy', allow_pickle = True)\n","train_label = np.load(BASE_PATH+'/train_label.npy', allow_pickle = True)\n","test_img = np.load(BASE_PATH+'/test_img_seq.npy', allow_pickle = True)\n","test_label = np.load(BASE_PATH+'/test_label.npy', allow_pickle = True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YoOX36je18Bn","executionInfo":{"status":"ok","timestamp":1623509935114,"user_tz":-540,"elapsed":263,"user":{"displayName":"‍이혜리[학생](소프트웨어융합대학 컴퓨터공학과)","photoUrl":"","userId":"16579190241035414660"}},"outputId":"3e639a0d-dc89-43dd-bec6-eedf404c6f93"},"source":["print(train_img.shape)\n","print(train_label.shape)\n","print(test_img.shape)\n","print(test_label.shape)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["(1994, 20, 97, 173, 3)\n","(1994, 3)\n","(401, 20, 97, 173, 3)\n","(401, 3)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"xt6UbAh03aRj"},"source":["# fit and evaluate a model\n","def evaluate_model(trainX, trainy, testX, testy):\n","    epochs, batch_size = 10, 20\n","    n_steps = 20\n","    n_outputs = trainy.shape[1] # 3\n","\n","    # define model\n","    model = Sequential()\n","    model.add(ConvLSTM2D(filters=32, kernel_size=(3,3), activation='relu', data_format = \"channels_last\", input_shape=(n_steps, trainX.shape[2], trainX.shape[3], 3)))\n","    model.add(Dropout(0.5))\n","    model.add(Flatten())\n","    model.add(Dense(100, activation='relu'))\n","    model.add(Dense(n_outputs, activation='softmax'))\n","\n","    model.summary()\n","\n","    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', tf.keras.metrics.Recall(), tf.keras.metrics.Precision()])\n","    # fit network\n","    model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size, verbose=1, validation_split=0.15, shuffle=True)\n","    # model save\n","    model.save(BASE_PATH+'/convLSTM.h5')\n","\t# evaluate model\n","    accuracy = model.evaluate(testX, testy, batch_size=batch_size, verbose=1)\n","    return accuracy\n","\n","acc = evaluate_model(train_img, train_label, test_img, test_label)\n","print('test accuracy: %.2f%%', % (acc[1]*100))"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"QCjP5lXs_Loe"},"source":["### 데이터 시험해보기"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":942,"output_embedded_package_id":"1IWQL7D1th7TXlyWnXlZ5a6wDWOo7I5Wr"},"id":"yGoPognn4hC3","executionInfo":{"status":"ok","timestamp":1623503736985,"user_tz":-540,"elapsed":15093,"user":{"displayName":"‍이혜리[학생](소프트웨어융합대학 컴퓨터공학과)","photoUrl":"","userId":"16579190241035414660"}},"outputId":"c4e5ee26-6e99-4ccb-9fb7-145b614d1ba5"},"source":["model = load_model(BASE_PATH+'/convLSTM.h5')\n","img_number = 400\n","sc = model.evaluate(test_img[img_number:img_number+1], test_label[img_number:img_number+1], verbose=0)\n","print(\"%s: %.2f%%\" % (model.metrics_names[1], sc[1]*100))\n","print(\"실제 label: \", test_label[img_number])\n","\n","fig=plt.figure(figsize=(27, 13))\n","rows = 4\n","cols = 5\n","for i in range(20):\n","    im = cv2.resize(test_img[img_number][i], dsize=(320, 180), interpolation=cv2.INTER_LINEAR)\n","    fig.add_subplot(rows, cols, i+1)\n","    plt.imshow(im)\n","\n","plt.tight_layout()\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"Output hidden; open in https://colab.research.google.com to view."},"metadata":{}}]}]}