2020-2-capstone-design2 / 2017110275

Go to a project
Toggle navigation Toggle navigation pinning
이한솔
af6c5a96 1 parent 3c03fa8f

Add CNN

Inline Side-by-side
Showing 3 changed files with 1432 additions and 0 deletions
{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"import csv\n",
"import numpy as np\n",
"import os\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"def transferData(dir_, outputdir_) :\n",
" outputfile = open(outputdir_, 'w', newline='\\n')\n",
" fopen = open(dir_, 'r')\n",
" csvreader = csv.reader(fopen)\n",
" \n",
" fw = csv.writer(outputfile)\n",
" \n",
" for line in csvreader :\n",
" newdata = []\n",
" can_id = int(line[1], 16)\n",
" newdata.append(can_id)\n",
" newdata.append(line[2])\n",
" \n",
" ll = len(line)\n",
" num = int(line[2])\n",
" \n",
" data = []\n",
" if num == 8 :\n",
" data = line[3:11]\n",
" else :\n",
" data = line[3:3+num]\n",
" for _ in range(8-num):\n",
" data.append('0')\n",
" #print(line)\n",
" newdata.extend(list(map(lambda x:int(x, 16), data)))\n",
" \n",
" if line[-1] == 'T' :\n",
" newdata.append(0)\n",
" else :\n",
" newdata.append(1)\n",
" \n",
" fw.writerow(newdata)\n",
"\n",
"def Preprocess(dir_) :\n",
" f1 = open(dir_, 'r')\n",
" fr = csv.reader(f1)\n",
" d = [] \n",
" for line in fr :\n",
" l = []\n",
" l.append(line[0])\n",
" l.append(line[1])\n",
" l.extend(line[2:11])\n",
" \n",
" d.append(l)\n",
" return d"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"transferData('DoS_dataset.csv', 'DoS.csv')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"datalist = Preprocess('DoS.csv')"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"datalist = datalist[0:3665761]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['790', '8', '5', '33', '104', '9', '33', '33', '0', '111', '1']"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"datalist[0]"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"def createsequence(data) :\n",
" arr = []\n",
" for i in range(0, len(data)-11): \n",
" d = []\n",
" t = []\n",
" for x in range(i, i+11):\n",
" t = list(map(float, data[x]))\n",
" #t = np.array(t, dtype=np.float32)\n",
" #t = tf.convert_to_tensor(t)\n",
" d.append(t)\n",
" arr.append(d)\n",
" return arr"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"resData = createsequence(datalist)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[0.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [305.0, 8.0, 0.0, 128.0, 0.0, 0.0, 76.0, 127.0, 10.0, 154.0, 1.0], [0.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 37.0, 42.0, 71.0, 1.0], [0.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [880.0, 8.0, 0.0, 32.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0], [0.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1087.0, 8.0, 0.0, 64.0, 96.0, 255.0, 126.0, 203.0, 8.0, 0.0, 1.0], [0.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 203.0, 8.0, 0.0, 1.0], [0.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]\n",
"11\n",
"<class 'list'>\n"
]
}
],
"source": [
"print(resData[1478])\n",
"print(len(resData[1478]))\n",
"print(type(resData))"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"def check(data):\n",
" check = []\n",
" \n",
" for e in data :\n",
" flag = False\n",
" for i in e :\n",
" if i[-1] == 0 :\n",
" check.append(0)\n",
" flag = True\n",
" break\n",
" if flag == False :\n",
" check.append(1)\n",
" \n",
" return check\n",
"\n",
"cc = check(resData)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2585455\n",
"1080295\n"
]
}
],
"source": [
"cnt = 0\n",
"for e in cc:\n",
" if e == 1 :\n",
" cnt += 1\n",
" \n",
"print(cnt)\n",
"print(len(cc)-cnt)"
]
},
{
"cell_type": "code",
"execution_count": 216,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"import tensorflow as tf"
]
},
{
"cell_type": "code",
"execution_count": 404,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"df = pd.DataFrame(resData)\n",
"n = len(df)\n",
"\n",
"X_train = df.iloc[0:n, 0:11].values.tolist()\n",
"Y_train = cc\n",
"\n",
"#X_train = pd.\n",
"X_train = np.asarray(X_train)\n",
"Y_train = np.asarray(Y_train)\n",
"#X_train = tf.constant(X_train)\n",
"#print(X_train[-1])\n",
"#Y_train = tf.convert_to_tensor(Y_train)\n",
"\n",
"#Y_train = np.squeeze(Y_train, axis=1)\n",
"x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3, random_state=321)"
]
},
{
"cell_type": "code",
"execution_count": 405,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(333251, 11)\n",
"(3665750, 11, 11)\n",
"(3665750,)\n"
]
}
],
"source": [
"print(X_train_.shape)\n",
"print(X_train.shape)\n",
"print(Y_train.shape)"
]
},
{
"cell_type": "code",
"execution_count": 241,
"metadata": {},
"outputs": [],
"source": [
"for i in x_train:\n",
" for j in i:\n",
" if len(j) != 11:\n",
" print(j)"
]
},
{
"cell_type": "code",
"execution_count": 211,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" <th>7</th>\n",
" <th>8</th>\n",
" <th>9</th>\n",
" <th>10</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>[790.0, 8.0, 5.0, 33.0, 104.0, 9.0, 33.0, 33.0...</td>\n",
" <td>[399.0, 8.0, 254.0, 91.0, 0.0, 0.0, 0.0, 60.0,...</td>\n",
" <td>[608.0, 8.0, 25.0, 33.0, 34.0, 48.0, 8.0, 142....</td>\n",
" <td>[672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2...</td>\n",
" <td>[809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3...</td>\n",
" <td>[1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0...</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128...</td>\n",
" <td>[305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>[399.0, 8.0, 254.0, 91.0, 0.0, 0.0, 0.0, 60.0,...</td>\n",
" <td>[608.0, 8.0, 25.0, 33.0, 34.0, 48.0, 8.0, 142....</td>\n",
" <td>[672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2...</td>\n",
" <td>[809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3...</td>\n",
" <td>[1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0...</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128...</td>\n",
" <td>[305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127...</td>\n",
" <td>[320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>[608.0, 8.0, 25.0, 33.0, 34.0, 48.0, 8.0, 142....</td>\n",
" <td>[672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2...</td>\n",
" <td>[809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3...</td>\n",
" <td>[1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0...</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128...</td>\n",
" <td>[305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127...</td>\n",
" <td>[320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 20.0, 104.0, 119.0, 0....</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>[672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2...</td>\n",
" <td>[809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3...</td>\n",
" <td>[1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0...</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128...</td>\n",
" <td>[305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127...</td>\n",
" <td>[320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 20.0, 104.0, 119.0, 0....</td>\n",
" <td>[1087.0, 8.0, 0.0, 64.0, 96.0, 255.0, 126.0, 2...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>[809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3...</td>\n",
" <td>[1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0...</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128...</td>\n",
" <td>[305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127...</td>\n",
" <td>[320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 20.0, 104.0, 119.0, 0....</td>\n",
" <td>[1087.0, 8.0, 0.0, 64.0, 96.0, 255.0, 126.0, 2...</td>\n",
" <td>[880.0, 8.0, 0.0, 32.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>333246</th>\n",
" <td>[809.0, 8.0, 220.0, 183.0, 127.0, 20.0, 17.0, ...</td>\n",
" <td>[880.0, 8.0, 0.0, 32.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[1072.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138...</td>\n",
" <td>[1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0...</td>\n",
" <td>[497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0...</td>\n",
" <td>[1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>333247</th>\n",
" <td>[880.0, 8.0, 0.0, 32.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[1072.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138...</td>\n",
" <td>[1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0...</td>\n",
" <td>[497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0...</td>\n",
" <td>[1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1...</td>\n",
" <td>[305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>333248</th>\n",
" <td>[1072.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
" <td>[1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138...</td>\n",
" <td>[1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0...</td>\n",
" <td>[497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0...</td>\n",
" <td>[1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1...</td>\n",
" <td>[305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127...</td>\n",
" <td>[320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 22.0, 34.0, 4...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>333249</th>\n",
" <td>[1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138...</td>\n",
" <td>[1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0...</td>\n",
" <td>[497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0...</td>\n",
" <td>[1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1...</td>\n",
" <td>[305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127...</td>\n",
" <td>[320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 22.0, 34.0, 4...</td>\n",
" <td>[1264.0, 8.0, 0.0, 0.0, 0.0, 128.0, 0.0, 105.0...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>333250</th>\n",
" <td>[1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0...</td>\n",
" <td>[497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...</td>\n",
" <td>[339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255....</td>\n",
" <td>[2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ...</td>\n",
" <td>[848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0...</td>\n",
" <td>[1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0....</td>\n",
" <td>[304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1...</td>\n",
" <td>[305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127...</td>\n",
" <td>[320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 22.0, 34.0, 4...</td>\n",
" <td>[1264.0, 8.0, 0.0, 0.0, 0.0, 128.0, 0.0, 105.0...</td>\n",
" <td>[1680.0, 8.0, 0.0, 0.0, 1.0, 0.0, 160.0, 34.0,...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>333251 rows × 11 columns</p>\n",
"</div>"
],
"text/plain": [
" 0 \\\n",
"0 [790.0, 8.0, 5.0, 33.0, 104.0, 9.0, 33.0, 33.0... \n",
"1 [399.0, 8.0, 254.0, 91.0, 0.0, 0.0, 0.0, 60.0,... \n",
"2 [608.0, 8.0, 25.0, 33.0, 34.0, 48.0, 8.0, 142.... \n",
"3 [672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2... \n",
"4 [809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3... \n",
"... ... \n",
"333246 [809.0, 8.0, 220.0, 183.0, 127.0, 20.0, 17.0, ... \n",
"333247 [880.0, 8.0, 0.0, 32.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"333248 [1072.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"333249 [1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138... \n",
"333250 [1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0... \n",
"\n",
" 1 \\\n",
"0 [399.0, 8.0, 254.0, 91.0, 0.0, 0.0, 0.0, 60.0,... \n",
"1 [608.0, 8.0, 25.0, 33.0, 34.0, 48.0, 8.0, 142.... \n",
"2 [672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2... \n",
"3 [809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3... \n",
"4 [1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0... \n",
"... ... \n",
"333246 [880.0, 8.0, 0.0, 32.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"333247 [1072.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"333248 [1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138... \n",
"333249 [1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0... \n",
"333250 [497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n",
"\n",
" 2 \\\n",
"0 [608.0, 8.0, 25.0, 33.0, 34.0, 48.0, 8.0, 142.... \n",
"1 [672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2... \n",
"2 [809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3... \n",
"3 [1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0... \n",
"4 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,... \n",
"... ... \n",
"333246 [1072.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"333247 [1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138... \n",
"333248 [1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0... \n",
"333249 [497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n",
"333250 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"\n",
" 3 \\\n",
"0 [672.0, 8.0, 100.0, 0.0, 154.0, 29.0, 151.0, 2... \n",
"1 [809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3... \n",
"2 [1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0... \n",
"3 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,... \n",
"4 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"... ... \n",
"333246 [1088.0, 8.0, 255.0, 0.0, 0.0, 0.0, 255.0, 138... \n",
"333247 [1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0... \n",
"333248 [497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n",
"333249 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"333250 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ... \n",
"\n",
" 4 \\\n",
"0 [809.0, 8.0, 64.0, 187.0, 127.0, 20.0, 17.0, 3... \n",
"1 [1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0... \n",
"2 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,... \n",
"3 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"4 [704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"... ... \n",
"333246 [1201.0, 8.0, 41.0, 39.0, 39.0, 35.0, 0.0, 0.0... \n",
"333247 [497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n",
"333248 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"333249 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ... \n",
"333250 [848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0... \n",
"\n",
" 5 \\\n",
"0 [1349.0, 8.0, 216.0, 0.0, 0.0, 138.0, 0.0, 0.0... \n",
"1 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,... \n",
"2 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"3 [704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"4 [304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128... \n",
"... ... \n",
"333246 [497.0, 8.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n",
"333247 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"333248 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ... \n",
"333249 [848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0... \n",
"333250 [1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0.... \n",
"\n",
" 6 \\\n",
"0 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 11.0,... \n",
"1 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"2 [704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"3 [304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128... \n",
"4 [305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127... \n",
"... ... \n",
"333246 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"333247 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ... \n",
"333248 [848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0... \n",
"333249 [1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0.... \n",
"333250 [304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1... \n",
"\n",
" 7 \\\n",
"0 [339.0, 8.0, 0.0, 33.0, 16.0, 255.0, 0.0, 255.... \n",
"1 [704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"2 [304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128... \n",
"3 [305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127... \n",
"4 [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43... \n",
"... ... \n",
"333246 [2.0, 8.0, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 8.0, ... \n",
"333247 [848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0... \n",
"333248 [1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0.... \n",
"333249 [304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1... \n",
"333250 [305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127... \n",
"\n",
" 8 \\\n",
"0 [704.0, 8.0, 20.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"1 [304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128... \n",
"2 [305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127... \n",
"3 [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43... \n",
"4 [848.0, 8.0, 5.0, 32.0, 20.0, 104.0, 119.0, 0.... \n",
"... ... \n",
"333246 [848.0, 8.0, 5.0, 32.0, 164.0, 104.0, 118.0, 0... \n",
"333247 [1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0.... \n",
"333248 [304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1... \n",
"333249 [305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127... \n",
"333250 [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 22.0, 34.0, 4... \n",
"\n",
" 9 \\\n",
"0 [304.0, 8.0, 8.0, 128.0, 0.0, 255.0, 49.0, 128... \n",
"1 [305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127... \n",
"2 [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43... \n",
"3 [848.0, 8.0, 5.0, 32.0, 20.0, 104.0, 119.0, 0.... \n",
"4 [1087.0, 8.0, 0.0, 64.0, 96.0, 255.0, 126.0, 2... \n",
"... ... \n",
"333246 [1349.0, 8.0, 216.0, 82.0, 0.0, 139.0, 0.0, 0.... \n",
"333247 [304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1... \n",
"333248 [305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127... \n",
"333249 [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 22.0, 34.0, 4... \n",
"333250 [1264.0, 8.0, 0.0, 0.0, 0.0, 128.0, 0.0, 105.0... \n",
"\n",
" 10 \n",
"0 [305.0, 8.0, 229.0, 127.0, 0.0, 0.0, 72.0, 127... \n",
"1 [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 8.0, 34.0, 43... \n",
"2 [848.0, 8.0, 5.0, 32.0, 20.0, 104.0, 119.0, 0.... \n",
"3 [1087.0, 8.0, 0.0, 64.0, 96.0, 255.0, 126.0, 2... \n",
"4 [880.0, 8.0, 0.0, 32.0, 0.0, 0.0, 0.0, 0.0, 0.... \n",
"... ... \n",
"333246 [304.0, 8.0, 17.0, 128.0, 0.0, 255.0, 251.0, 1... \n",
"333247 [305.0, 8.0, 233.0, 127.0, 0.0, 0.0, 63.0, 127... \n",
"333248 [320.0, 8.0, 0.0, 0.0, 0.0, 0.0, 22.0, 34.0, 4... \n",
"333249 [1264.0, 8.0, 0.0, 0.0, 0.0, 128.0, 0.0, 105.0... \n",
"333250 [1680.0, 8.0, 0.0, 0.0, 1.0, 0.0, 160.0, 34.0,... \n",
"\n",
"[333251 rows x 11 columns]"
]
},
"execution_count": 211,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df"
]
},
{
"cell_type": "code",
"execution_count": 406,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(1099725, 11, 11)\n"
]
}
],
"source": [
"print(x_test.shape)"
]
},
{
"cell_type": "code",
"execution_count": 358,
"metadata": {},
"outputs": [],
"source": [
"# Model\n",
"from keras.models import Sequential\n",
"from keras.layers import Dense, LSTM, Conv1D, Dropout, GlobalAveragePooling1D, MaxPooling1D\n",
"from keras.utils import np_utils\n",
"from keras.layers.normalization import BatchNormalization"
]
},
{
"cell_type": "code",
"execution_count": 407,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_53\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"conv1d_99 (Conv1D) (None, 9, 64) 2176 \n",
"_________________________________________________________________\n",
"global_average_pooling1d_24 (None, 64) 0 \n",
"_________________________________________________________________\n",
"dropout_13 (Dropout) (None, 64) 0 \n",
"_________________________________________________________________\n",
"dense_35 (Dense) (None, 1) 65 \n",
"=================================================================\n",
"Total params: 2,241\n",
"Trainable params: 2,241\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"model = Sequential()\n",
"model.add(Conv1D(64, 3, input_shape=(11, 11), activation='relu'))\n",
"#model.add(Conv1D(64, 3, activation='relu'))\n",
"#model.add(MaxPooling1D(2))\n",
"#model.add(Conv1D(128, 2, activation='relu'))\n",
"#model.add(Conv1D(128, 2, activation='relu'))\n",
"model.add(GlobalAveragePooling1D())\n",
"model.add(Dropout(0.5))\n",
"model.add(Dense(1, activation='sigmoid'))\n",
"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": 408,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/10\n",
"25661/25661 [==============================] - 33s 1ms/step - loss: 0.0926 - accuracy: 0.9975 - val_loss: 0.0021 - val_accuracy: 0.9997\n",
"Epoch 2/10\n",
"25661/25661 [==============================] - 32s 1ms/step - loss: 0.0019 - accuracy: 0.9997 - val_loss: 8.2212e-04 - val_accuracy: 0.9998\n",
"Epoch 3/10\n",
"25661/25661 [==============================] - 33s 1ms/step - loss: 0.0012 - accuracy: 0.9998 - val_loss: 5.0894e-04 - val_accuracy: 0.9999\n",
"Epoch 4/10\n",
"25661/25661 [==============================] - 33s 1ms/step - loss: 8.3683e-04 - accuracy: 0.9999 - val_loss: 4.0702e-04 - val_accuracy: 0.9999\n",
"Epoch 5/10\n",
"25661/25661 [==============================] - 32s 1ms/step - loss: 6.7326e-04 - accuracy: 0.9999 - val_loss: 1.0525e-04 - val_accuracy: 1.0000\n",
"Epoch 6/10\n",
"25661/25661 [==============================] - 33s 1ms/step - loss: 4.6771e-04 - accuracy: 0.9999 - val_loss: 8.2441e-05 - val_accuracy: 1.0000\n",
"Epoch 7/10\n",
"25661/25661 [==============================] - 32s 1ms/step - loss: 5.0164e-04 - accuracy: 0.9999 - val_loss: 2.7738e-05 - val_accuracy: 1.0000\n",
"Epoch 8/10\n",
"25661/25661 [==============================] - 32s 1ms/step - loss: 4.3489e-04 - accuracy: 0.9999 - val_loss: 5.0454e-05 - val_accuracy: 1.0000\n",
"Epoch 9/10\n",
"25661/25661 [==============================] - 32s 1ms/step - loss: 3.8201e-04 - accuracy: 0.9999 - val_loss: 3.5519e-05 - val_accuracy: 1.0000\n",
"Epoch 10/10\n",
"25661/25661 [==============================] - 32s 1ms/step - loss: 2.5039e-04 - accuracy: 1.0000 - val_loss: 3.4685e-05 - val_accuracy: 1.0000\n"
]
}
],
"source": [
"history = model.fit(x_train, y_train, batch_size=100, epochs=10,validation_data=(x_test, y_test))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import csv\n",
"import numpy as np\n",
"import os\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def Preprocess(dir_) :\n",
" f1 = open(dir_, 'r')\n",
" fr = csv.reader(f1)\n",
" d = [] \n",
" for line in fr :\n",
" l = []\n",
" l.append(line[0])\n",
" l.append(line[1])\n",
" l.extend(line[2:11])\n",
" \n",
" d.append(l)\n",
" return d\n",
"\n",
"data = Preprocess('Fuzzy.csv')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"data = data[:3838857]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['1349', '8', '216', '0', '0', '138', '0', '0', '0', '0', '1']\n"
]
}
],
"source": [
"print(data[0])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def createsequence(data) :\n",
" arr = []\n",
" for i in range(0, len(data)-11): \n",
" d = []\n",
" t = []\n",
" for x in range(i, i+11):\n",
" t = list(map(float, data[x]))\n",
" #t = np.array(t, dtype=np.float32)\n",
" #t = tf.convert_to_tensor(t)\n",
" d.append(t)\n",
" arr.append(d)\n",
" return arr\n",
"\n",
"resdata = createsequence(data)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"def check(data):\n",
" check = []\n",
" \n",
" for e in data :\n",
" flag = False\n",
" for i in e :\n",
" if i[-1] == 0 :\n",
" check.append(0)\n",
" flag = True\n",
" break\n",
" if flag == False :\n",
" check.append(1)\n",
" \n",
" return check\n",
"\n",
"checkblock = check(resdata)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2582020\n",
"1256826\n"
]
}
],
"source": [
"cnt = 0\n",
"for e in checkblock :\n",
" if e == 1 :\n",
" cnt += 1\n",
"\n",
"print(cnt)\n",
"print(len(checkblock)-cnt)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"import tensorflow as tf\n",
"import pandas as pd\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"df = pd.DataFrame(resdata)\n",
"n = len(df)\n",
"\n",
"X_train = df.iloc[0:n, 0:11].values.tolist()\n",
"Y_train = checkblock\n",
"\n",
"X_train = np.asarray(X_train)\n",
"Y_train = np.asarray(Y_train)\n",
"\n",
"x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3, random_state=321)"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
"for i in x_train:\n",
" for j in i:\n",
" if len(j) != 11:\n",
" print(j)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(1151654, 11, 11)\n"
]
}
],
"source": [
"print(x_test.shape)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# Model\n",
"from keras.models import Sequential\n",
"from keras.layers import Dense, LSTM, Conv1D, Dropout, GlobalAveragePooling1D, MaxPooling1D\n",
"from keras.utils import np_utils\n",
"from tensorflow.keras.utils import plot_model\n",
"from keras.layers.normalization import BatchNormalization"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"conv1d (Conv1D) (None, 9, 64) 2176 \n",
"_________________________________________________________________\n",
"global_average_pooling1d (Gl (None, 64) 0 \n",
"_________________________________________________________________\n",
"dropout (Dropout) (None, 64) 0 \n",
"_________________________________________________________________\n",
"dense (Dense) (None, 1) 65 \n",
"=================================================================\n",
"Total params: 2,241\n",
"Trainable params: 2,241\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"('Failed to import pydot. You must `pip install pydot` and install graphviz (https://graphviz.gitlab.io/download/), ', 'for `pydotprint` to work.')\n"
]
}
],
"source": [
"model = Sequential()\n",
"model.add(Conv1D(64, 3, input_shape=(11, 11), activation='relu'))\n",
"model.add(GlobalAveragePooling1D())\n",
"model.add(Dropout(0.5))\n",
"model.add(Dense(1, activation='sigmoid'))\n",
"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
"model.summary()\n",
"\n",
"plot_model(model, to_file='CNNmodel.png')"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.1271 - accuracy: 0.9865 - val_loss: 0.0066 - val_accuracy: 0.9983\n",
"Epoch 2/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0129 - accuracy: 0.9969 - val_loss: 0.0060 - val_accuracy: 0.9986\n",
"Epoch 3/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0107 - accuracy: 0.9975 - val_loss: 0.0046 - val_accuracy: 0.9990\n",
"Epoch 4/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0093 - accuracy: 0.9979 - val_loss: 0.0037 - val_accuracy: 0.9993\n",
"Epoch 5/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0081 - accuracy: 0.9982 - val_loss: 0.0034 - val_accuracy: 0.9993\n",
"Epoch 6/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0074 - accuracy: 0.9984 - val_loss: 0.0042 - val_accuracy: 0.9991\n",
"Epoch 7/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0070 - accuracy: 0.9985 - val_loss: 0.0039 - val_accuracy: 0.9992\n",
"Epoch 8/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0065 - accuracy: 0.9987 - val_loss: 0.0033 - val_accuracy: 0.9994\n",
"Epoch 9/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0063 - accuracy: 0.9987 - val_loss: 0.0032 - val_accuracy: 0.9994\n",
"Epoch 10/10\n",
"26872/26872 [==============================] - 31s 1ms/step - loss: 0.0058 - accuracy: 0.9988 - val_loss: 0.0031 - val_accuracy: 0.9994\n"
]
}
],
"source": [
"history = model.fit(x_train, y_train, batch_size=100, epochs=10,validation_data=(x_test, y_test))"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"35990/35990 [==============================] - 13s 361us/step - loss: 0.0031 - accuracy: 0.9994\n",
"\n",
" Test accuracy : 0.9994\n"
]
}
],
"source": [
"print(\"\\n Test accuracy : %.4f\"%(model.evaluate(x_test, y_test)[1]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import csv\n",
"import numpy as np\n",
"import os\n",
"import pandas as pd\n",
"import import_ipynb\n",
"import DoS_CNN as dos"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dos.transferData('gear_dataset.csv', 'Spoofing.csv')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"data = dos.Preprocess('Spoofing.csv')"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['320', '8', '0', '0', '0', '0', '16', '41', '42', '36', '1']"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[0]"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"def createsequence(data) :\n",
" arr = []\n",
" for i in range(0, len(data)-11): \n",
" d = []\n",
" t = []\n",
" for x in range(i, i+11):\n",
" t = list(map(float, data[x]))\n",
" d.append(t)\n",
" arr.append(d)\n",
" return arr\n",
"\n",
"resdata = createsequence(data)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"def check(data):\n",
" check = []\n",
" \n",
" for e in data :\n",
" flag = False\n",
" for i in e :\n",
" if i[-1] == 0 :\n",
" check.append(0)\n",
" flag = True\n",
" break\n",
" if flag == False :\n",
" check.append(1)\n",
" \n",
" return check\n",
"\n",
"checkblock = check(resdata)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2580434\n",
"1862697\n"
]
}
],
"source": [
"cnt = 0 # 1\n",
"for i in checkblock :\n",
" if i == 1 :\n",
" cnt += 1\n",
"print(cnt) # normal message\n",
"print(len(checkblock)-cnt) #abnormal"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"import tensorflow as tf\n",
"import pandas as pd\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"df = pd.DataFrame(resdata)\n",
"n = len(df)\n",
"\n",
"X_train = df.iloc[0:n, 0:11].values.tolist()\n",
"Y_train = checkblock\n",
"\n",
"X_train = np.asarray(X_train)\n",
"Y_train = np.asarray(Y_train)\n",
"\n",
"x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3, random_state=321)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"for i in x_train:\n",
" for j in i:\n",
" if len(j) != 11:\n",
" print(j)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(1332940, 11, 11)\n"
]
}
],
"source": [
"print(x_test.shape)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"# Model\n",
"from keras.models import Sequential\n",
"from keras.layers import Dense, LSTM, Conv1D, Dropout, GlobalAveragePooling1D, MaxPooling1D\n",
"from keras.utils import np_utils\n",
"from keras.layers.normalization import BatchNormalization"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"conv1d (Conv1D) (None, 9, 64) 2176 \n",
"_________________________________________________________________\n",
"global_average_pooling1d (Gl (None, 64) 0 \n",
"_________________________________________________________________\n",
"dropout (Dropout) (None, 64) 0 \n",
"_________________________________________________________________\n",
"dense (Dense) (None, 1) 65 \n",
"=================================================================\n",
"Total params: 2,241\n",
"Trainable params: 2,241\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"model = Sequential()\n",
"model.add(Conv1D(64, 3, input_shape=(11, 11), activation='relu'))\n",
"model.add(GlobalAveragePooling1D())\n",
"model.add(Dropout(0.5))\n",
"model.add(Dense(1, activation='sigmoid'))\n",
"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0739 - accuracy: 0.9938 - val_loss: 6.8605e-04 - val_accuracy: 1.0000\n",
"Epoch 2/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0029 - accuracy: 0.9995 - val_loss: 2.1258e-04 - val_accuracy: 1.0000\n",
"Epoch 3/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0030 - accuracy: 0.9995 - val_loss: 2.5308e-04 - val_accuracy: 1.0000\n",
"Epoch 4/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0025 - accuracy: 0.9996 - val_loss: 9.4737e-05 - val_accuracy: 1.0000\n",
"Epoch 5/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0031 - accuracy: 0.9995 - val_loss: 4.3025e-04 - val_accuracy: 1.0000\n",
"Epoch 6/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0022 - accuracy: 0.9997 - val_loss: 6.6521e-05 - val_accuracy: 1.0000\n",
"Epoch 7/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0020 - accuracy: 0.9997 - val_loss: 7.7669e-05 - val_accuracy: 1.0000\n",
"Epoch 8/10\n",
"31102/31102 [==============================] - 36s 1ms/step - loss: 0.0019 - accuracy: 0.9997 - val_loss: 1.3602e-04 - val_accuracy: 1.0000\n",
"Epoch 9/10\n",
"31102/31102 [==============================] - 35s 1ms/step - loss: 0.0018 - accuracy: 0.9997 - val_loss: 3.1278e-05 - val_accuracy: 1.0000\n",
"Epoch 10/10\n",
"31102/31102 [==============================] - 35s 1ms/step - loss: 0.0017 - accuracy: 0.9998 - val_loss: 3.1277e-05 - val_accuracy: 1.0000\n"
]
}
],
"source": [
"history = model.fit(x_train, y_train, batch_size=100, epochs=10,validation_data=(x_test, y_test))"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"41655/41655 [==============================] - 15s 352us/step - loss: 3.1277e-05 - accuracy: 1.0000\n",
"\n",
" Test accuracy : 1.0000\n"
]
}
],
"source": [
"print(\"\\n Test accuracy : %.4f\"%(model.evaluate(x_test, y_test)[1]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}