video_vector_generator.py
2.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import pandas as pd
import numpy as np
from gensim.models import Word2Vec
import activation as ac
BATCH_SIZE = 1000
def normalize_tag(tag):
if isinstance(tag, str):
new_tag = tag.lower().replace('[^a-zA-Z]', ' ')
if new_tag.find(" (") != -1:
new_tag = new_tag[:new_tag.find(" (")]
new_tag = new_tag.replace(" ", "-")
new_tag = new_tag.replace('"', "")
return new_tag
else:
return tag
def vectorization_video():
print('[0.1 0.2]')
if __name__ == '__main__':
tag_vectors = Word2Vec.load("./tag_vectors.model").wv
video_vectors = Word2Vec().wv # Empty model
# Load video recommendation tags.
video_tags = pd.read_csv('./segment_tags.csv', encoding='utf-8',error_bad_lines=False)
# Define batch variables.
batch_video_ids = []
batch_video_vectors = []
error_tags = []
for i, row in video_tags.iterrows():
#print(row)
video_id = row[0]
video_vector = np.zeros(100)
for segment_index in range(1, 6):
#get tag and weight from here
#use weight as input of non-linear function before this step
#print("SEG TAG ",row[segment_index])
tag, weight = row[segment_index].split(":")
#print(tag)
if tag in tag_vectors.vocab:
video_vector = video_vector + (tag_vectors[tag] * float(weight))
else:
#print("unknown", tag)
# Pass if tag is unknown
if tag not in error_tags:
error_tags.append(tag)
batch_video_ids.append(video_id)
batch_video_vectors.append(video_vector)
# Add video vectors.
if (i+1) % BATCH_SIZE == 0:
video_vectors.add(batch_video_ids, batch_video_vectors)
batch_video_ids = []
batch_video_vectors = []
print("Video vectors created: ", i+1)
# Add rest of video vectors.
video_vectors.add(batch_video_ids, batch_video_vectors)
print("error tags: ")
print(error_tags)
print(len(error_tags))
video_vectors.save("video_vectors.model")
# Usage
# video_vectors = Word2Vec().wv.load("video_vectors.model")
# video_vectors.most_similar("XwFj", topn=5)