2020-1-capstone-design2 / 2014104094

Go to a project
Toggle navigation Toggle navigation pinning
2014104094
9427d324 1 parent 73349ac1

파일 업로드 완료

Inline Side-by-side
Showing 20 changed files with 195 additions and 85 deletions
{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"AudioCutter.ipynb","provenance":[],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"sHHBf1bPIikR","colab_type":"code","outputId":"78f3777d-b504-419e-92d2-c7f8ec8ba3d7","executionInfo":{"status":"ok","timestamp":1591327904716,"user_tz":-540,"elapsed":55988,"user":{"displayName":"‍노경환[학생](소프트웨어융합대학 컴퓨터공학과)","photoUrl":"","userId":"12302221434260696963"}},"colab":{"base_uri":"https://localhost:8080/","height":121}},"source":["from google.colab import drive\n","drive.mount('/content/gdrive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/gdrive\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"Y57bL4vQX_q0","colab_type":"text"},"source":["## pydub 설치"]},{"cell_type":"code","metadata":{"id":"ijgvGXw2I79q","colab_type":"code","outputId":"e9310cd3-f1d9-4f74-ad2b-8df3ef2fada8","executionInfo":{"status":"ok","timestamp":1591327917988,"user_tz":-540,"elapsed":5733,"user":{"displayName":"‍노경환[학생](소프트웨어융합대학 컴퓨터공학과)","photoUrl":"","userId":"12302221434260696963"}},"colab":{"base_uri":"https://localhost:8080/","height":84}},"source":["pip install pydub"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Collecting pydub\n"," Downloading https://files.pythonhosted.org/packages/7b/d1/fbfa79371a8cd9bb15c2e3c480d7e6e340ed5cc55005174e16f48418333a/pydub-0.24.1-py2.py3-none-any.whl\n","Installing collected packages: pydub\n","Successfully installed pydub-0.24.1\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3q0weJwUMQNe","colab_type":"code","outputId":"a60a9368-0ed2-4012-cc2e-4c60b817746f","executionInfo":{"status":"ok","timestamp":1591328204085,"user_tz":-540,"elapsed":701,"user":{"displayName":"‍노경환[학생](소프트웨어융합대학 컴퓨터공학과)","photoUrl":"","userId":"12302221434260696963"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["cd /content/gdrive/Shared drives/반려견/데이터/sound/barking"],"execution_count":5,"outputs":[{"output_type":"stream","text":["/content/gdrive/Shared drives/반려견/데이터/sound/barking\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"yj1y1himXUGY","colab_type":"text"},"source":["## 오디오 편집 코드\n","> 오디오 편집 프로그램 Audacity를 이용하는 것보다는 편리하다\n","> 오디오 파일을 하나 밖에 부르지 못해서 파일 명과 label_num을 바꿔주어야 한다."]},{"cell_type":"code","metadata":{"id":"SG5iHu95JXNH","colab_type":"code","colab":{}},"source":["from pydub import AudioSegment\n","\n","def cutAudio(input_file, maxtime):\n"," t1 = 0\n"," t2 = t1+10 # 10초 단위로 끊음\n"," label_num = 5867\n","\n"," while t1 < maxtime: # t1이 전체 오디오 시간보다 작으면 계속 반복\n"," # pydub은 miliseconds 단위 사용 (1000milis -> 1초)\n"," start = t1 * 1000 # 편집할 오디오 시작 시간\n"," end = t2 * 1000 # 편집할 오디오 끝나는 시간\n"," newAudio = AudioSegment.from_wav(input_file)\n"," newAudio = newAudio[start:end]\n"," newAudio = newAudio.set_channels(1) # 오디오 채널 1개로 만듦\n"," newAudio.export('/content/gdrive/Shared drives/반려견/데이터/sound_data/barking/'+'barking_'+str(label_num)+'.wav', format=\"wav\") # 새로운 형태로 export\n"," label_num += 1 # 라벨 번호 1증가\n"," t1 += 10\n"," t2 += 10\n","\n","# 메인 프로그램\n","cutAudio(\"barking_16.wav\", 7190)\n"],"execution_count":0,"outputs":[]}]}
\ No newline at end of file
import os
def changeName(path, name):
i = 1
for filename in os.listdir(path):
print(path+filename, '=>', path+str(name)+'_'+str(i)+'.jpg')
os.rename(path+filename, path+str(name)+'_'+str(i)+'.jpg')
i += 1
changeName('C:/Users/nokh9/Desktop/dog_sound_mel/barking/', 'barking')
#changeName('C:/Users/nokh9/Desktop/dog_sound_mel/lonely/', 'howling')
#changeName('C:/Users/nokh9/Desktop/dog_sound_mel/sad/', 'whimpering')
\ No newline at end of file
{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"make mel spectogram and image cut.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"ZvJROk8j0qup","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":125},"executionInfo":{"status":"ok","timestamp":1591441994343,"user_tz":-540,"elapsed":25504,"user":{"displayName":"‍김선우[ 대학원석·박사통합과정재학 / 컴퓨터학과 ]","photoUrl":"","userId":"10839284711926757044"}},"outputId":"b05a9479-9768-4c12-90c6-32fdeec84942"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"1InEz1YyLetk","colab_type":"text"},"source":["Sound data로부터 Mel_spectogram plot"]},{"cell_type":"code","metadata":{"id":"N5FoxNMJ51w8","colab_type":"code","colab":{}},"source":["import os\n","import librosa\n","import librosa.display\n","import IPython.display\n","import numpy as np\n","import matplotlib.pyplot as plt\n","\n","%matplotlib inline\n","\n","audio_path = '../content/drive/Shared drives/반려견/데이터/sound_data'\n","specto_path = '../content/drive/Shared drives/반려견/데이터/mel_spectogram'\n","\n","barking_path = '/barking'\n","growling_path = '/growling'\n","howling_path = '/howling'\n","whimpering_path = '/whimpering'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"U2FIwn9055SX","colab_type":"code","colab":{}},"source":["def create_spectogram_image(sound_path):\n","\taudio_data_path = audio_path + sound_path\n","\tspecto_data_path = specto_path + sound_path\n","\taudio_list = os.listdir(audio_data_path)\n","\tlistdir_number = len(audio_list)\n","\n","\tFIG_SIZE = (5, 4)\n","\tframe_length = 0.025\n","\tframe_stride = 0.010\n","\n","\t\n","\tprint('-'*50)\n","\tprint('Creating spectogram images...'+sound_path)\n","\tprint('-'*50)\n","\t\n","\tfor number in range(4000):\n","\t\twav_file_path = audio_data_path + sound_path + '_' + str(number+1) + '.wav'\n","\t\tspecto_sava_path = specto_data_path + sound_path + '_' + str(number+1) + '.png'\n","\n","\t\tif os.path.isfile(wav_file_path):\n","\t\t\ty, sr = librosa.load(wav_file_path, sr=22500)\n","\t\t\tinput_nfft = int(round(sr*frame_length))\n","\t\t\tinput_stride = int(round(sr*frame_stride))\n","\n","\t\t\tS = librosa.feature.melspectrogram(y=y, n_mels=40, n_fft=input_nfft, hop_length=input_stride)\n","\n","\t\t\tplt.figure(figsize=FIG_SIZE)\n","\t\t\tlibrosa.display.specshow(librosa.power_to_db(S, ref=np.max), y_axis='mel', sr=sr, hop_length=input_stride, x_axis='time')\n","\t\t\tplt.axis('off'), plt.xticks([]), plt.yticks([])\n","\t\t\tplt.tight_layout()\n","\t\t\tplt.savefig(specto_sava_path,\n","\t bbox_inces='tight',\n","\t pad_inches=0\n","\t )\n","\t\t\tplt.close()\n","\t\t\tif((number+1)%100 == 0):\n","\t\t\t\tprint(str(number+1) + ' is finished')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Pimf_eZ0LV_a","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":738},"outputId":"90778a3d-72cd-4640-ab97-3fb700b6c0a6"},"source":["if __name__ == \"__main__\":\n","\tcreate_spectogram_image(barking_path)\n","\tcreate_spectogram_image(growling_path)\n","\tcreate_spectogram_image(howling_path)\n","\tcreate_spectogram_image(whimpering_path)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["--------------------------------------------------\n","Creating spectogram images.../barking\n","--------------------------------------------------\n","100 is finished\n","200 is finished\n","300 is finished\n","400 is finished\n","500 is finished\n","600 is finished\n","700 is finished\n","800 is finished\n","900 is finished\n","1000 is finished\n","1100 is finished\n","1200 is finished\n","1300 is finished\n","1400 is finished\n","1500 is finished\n","1600 is finished\n","1700 is finished\n","1800 is finished\n","1900 is finished\n","2000 is finished\n","2100 is finished\n","2200 is finished\n","2300 is finished\n","2400 is finished\n","2500 is finished\n","2600 is finished\n","2700 is finished\n","2800 is finished\n","2900 is finished\n","3000 is finished\n","3100 is finished\n","3200 is finished\n","3300 is finished\n","3400 is finished\n","3500 is finished\n","3600 is finished\n","3700 is finished\n","3800 is finished\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"m7LQ9yBRaVWW","colab_type":"text"},"source":["Plot한 Melspectogram 양 옆 공백 Cutting"]}]}
\ No newline at end of file
......@@ -27,8 +27,8 @@ from attention_module import cbam_block
@return y(ndarray) : Targets
"""
def load_data():
X = pickle.load(open("X_9.pickle", "rb"))
y = pickle.load(open("y_9.pickle", "rb"))
X = pickle.load(open("X_final2.pickle", "rb"))
y = pickle.load(open("y_final2.pickle", "rb"))
X = X/225.0
......@@ -75,12 +75,12 @@ def mobile_net(input_shape):
x = mobile_net_block(x, 1024, 2)
x = mobile_net_block(x, 1024)
#x = cbam_block(x)
x = cbam_block(x)
x = GlobalAvgPool2D()(x)
output = Dense(4, activation='softmax')(x)
output = Dense(3, activation='softmax')(x)
model = Model(input, output)
return model
......@@ -153,4 +153,4 @@ if __name__ == "__main__":
# predict sample
predict(model, X_to_predict, y_to_predict)
model.save('C:/Users/nokh9/Desktop/수업/캡스톤디자인2/소스코드/mobile_net3.h5')
\ No newline at end of file
model.save('C:/Users/nokh9/Desktop/mobile_net_final2.h5')
\ No newline at end of file
......
'''
https://m.blog.naver.com/PostView.nhn?blogId=jiwon6928&logNo=221185628702&proxyReferer=https:%2F%2Fwww.google.com%2F
'''
from PIL import Image, ImageDraw
import hcluster
import numpy as np
import os
from scipy.cluster.vq import *
import matplotlib.pyplot as plt
path = 'mel_spectogram_cut/barking'
#path = '/home/compu/ksw/dog_project/mel_spectogram_cut/barking'
imlist = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.png')]
features = np.zeros([len(imlist), 512])
for i, f in enumerate(imlist):
im = np.array(Image.open(f))
h, edge = np.histogramdd(im.reshape(-1,3), 8, normed=True,
range=[(0,255),(0,255),(0,255)])
features[i] = h.flatten()
if i%100 == 0:
print(i)
tree = hcluster.hcluster(features)
tree1 = hcluster.draw_dendrogram(tree, imlist)
\ No newline at end of file
from PIL import Image, ImageDraw
import hcluster
import numpy as np
import os
from scipy.cluster.vq import *
import matplotlib.pyplot as plt
#path = 'mel_spectogram_cut/barking'
path = 'C:/Users/nokh9/Desktop/mel_spectogram_cut/barking'
imlist = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.png')]
features = np.zeros([len(imlist), 512])
for i, f in enumerate(imlist):
im = np.array(Image.open(f))
h, edge = np.histogramdd(im.reshape(-1,3), 8, normed=True,
range=[(0,255),(0,255),(0,255)])
features[i] = h.flatten()
if i%100 == 0:
print(i)
tree = hcluster.hcluster(features)
tree1 = hcluster.draw_dendrogram(tree, imlist)
\ No newline at end of file
'''
https://m.blog.naver.com/PostView.nhn?blogId=jiwon6928&logNo=221185628702&proxyReferer=https:%2F%2Fwww.google.com%2F
'''
from itertools import combinations
import numpy as np
import math
from PIL import Image, ImageDraw
class ClusterNode(object):
def __init__(self, vec, left, right, distance=0.0, count=1):
self.left = left
self.right = right
self.vec = vec
self.distance = distance
self.count = count
def extract_clusters(self, dist):
if self.distance < dist:
return [self]
return self.left.extract_clusters(dist) + self.right.extract_clusters(dist)
def get_cluster_elements(self):
return self.left.get_cluster_elements() + self.right.get_cluster_elements()
def get_height(self):
return self.left.get_height() + self.right.get_height()
def get_depth(self):
return max(self.left.get_depth(), self.right.get_depth()) + self.distance
def draw(self, draw, x, y, s, imlist, im):
h1 = int(self.left.get_height() * 20 / 2)
h2 = int(self.right.get_height() * 20 / 2)
top = y - (h1 + h2)
bottom = y + (h1 + h2)
draw.line((x, top+h1, x, bottom-h2), fill=(0,0,0))
ll = self.distance*s
draw.line((x, top+h1, x+ll, top+h1), fill=(0,0,0))
draw.line((x, bottom-h2, x+ll, bottom-h2), fill=(0,0,0))
self.left.draw(draw, x+ll, top+h1, s, imlist, im)
self.right.draw(draw, x+ll, bottom-h2, s, imlist, im)
class ClusterLeafNode(object):
def __init__(self, vec, id):
self.vec = vec
self.id = id
def extract_clusters(self, dist):
return [self]
def get_cluster_elements(self):
return [self.id]
def get_height(self):
return 1
def get_depth(self):
return 0
def draw(self, draw, x, y, s, imlist, im):
nodeim = Image.open(imlist[self.id])
nodeim.thumbnail([20, 20])
ns = nodeim.size
im.paste(nodeim, [int(x), int(y-ns[1]//2), int(x+ns[0]), int(y+ns[1]-ns[1]//2)])
def L2dist(v1, v2):
return np.sqrt(sum((v1-v2)**2))
def L1dist(v1, v2):
return sum(abs(v1-v2))
def hcluster(features, distfcn=L2dist):
distances = {}
node = [ClusterLeafNode(np.array(f), id=i) for i, f in enumerate(features)]
while(len(node) > 1):
closest = float('inf')
for ni, nj in combinations(node, 2):
if(ni, nj) not in distances:
distances[ni, nj] = distfcn(ni.vec, nj.vec)
d = distances[ni, nj]
if d < closest:
closest = d
lowestpair = (ni, nj)
ni, nj = lowestpair
new_vec = (ni.vec + nj.vec) / 2.0
new_node = ClusterNode(new_vec, left=ni, right=nj, distance=closest)
node.remove(ni)
node.remove(nj)
node.append(new_node)
return node[0]
def draw_dendrogram(node, imlist, filename='clusters.jpg'):
rows = node.get_height() * 20
cols = 1200
s = float(cols-150) / node.get_depth()
im = Image.new('RGB', (cols, rows), (255,255,255))
draw = ImageDraw.Draw(im)
draw.line((0, rows/2, 20, rows/2), fill=(0,0,0))
node.draw(draw, 20, (rows/2), s, imlist, im)
im.save(filename)
im.show()
\ No newline at end of file
......@@ -9,8 +9,8 @@ import numpy as np
import matplotlib.pyplot as plt
import pickle
dataset_path = "C:/Users/nokh9/Desktop/dog_sound_mel/" # 데이터가 있는 경로
CATEGORIES = ["angry", "happy", "lonely", "sad"] # 감정에 대한 카테고리
dataset_path = "C:/Users/nokh9/Desktop/dog_sound_mel/mobile_net_" # 데이터가 있는 경로
CATEGORIES = ["barking", "howling", "whimpering"] # 감정에 대한 카테고리
training_data = []
......@@ -59,10 +59,10 @@ if __name__ == "__main__":
X = np.array(X).reshape(-1, 62, 78, 3)
# pickle 데이터로 x와 y를 따로 저장해줌
pickle_out = open("X_9.pickle", "wb")
pickle_out = open("X_final2.pickle", "wb")
pickle.dump(X, pickle_out)
pickle_out.close()
pickle_out = open("y_9.pickle", "wb")
pickle_out = open("y_final2.pickle", "wb")
pickle.dump(y, pickle_out)
pickle_out.close()
\ No newline at end of file
......
import cv2
import numpy as np
import sys
import os
import pyaudio
import librosa
import librosa.display
import matplotlib.pyplot as plt
import time
import tensorflow as tf
import tensorflow.keras as keras
rate = 22500
chunk_size = rate // 4
CATEGORIES = ["angry", "happy", "lonely", "sad"]
model= keras.models.load_model("C:/Users/nokh9/Desktop/mobile_net2.h5")
def prepare(mel):
img_array = cv2.imread(mel)
new_array = cv2.resize(img_array, (62, 78), 3)
return new_array.reshape(-1, 62, 78, 3)
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paFloat32,
channels=1,
rate=rate,
input=True,
input_device_index=1,
frames_per_buffer=chunk_size)
frames = []
do_melspec = librosa.feature.melspectrogram
pwr_to_db = librosa.core.power_to_db
"""
while True:
start = time.time()
data = stream.read(chunk_size)
data = np.fromstring(data, dtype=np.float32)
melspec = do_melspec(y=data, sr=rate, n_mels=128, fmax=4000)
norm_melspec = pwr_to_db(melspec, ref=np.max)
frames.append(norm_melspec)
if len(frames) == 20:
stack = np.hstack(frames)
plt.figure(figsize=(5, 4))
librosa.display.specshow(stack,fmax=4000)
plt.savefig('C:/Users/nokh9/Desktop/DB/' + 'db.jpg')
prediction = model.predict([prepare(r'C:/Users/nokh9/Desktop/DB/db.jpg')])
print(CATEGORIES[int(prediction[0][0])])
plt.draw()
plt.pause(0.0001)
plt.clf()
#break
frames.pop(0)
t = time.time() - start
print(1 / t)
"""
prediction = model.predict([prepare(r'C:/Users/nokh9/Desktop/DB/lonely_1.jpg')])
print(prediction)
\ No newline at end of file
No preview for this file type