강상위 / my-broadcasting

Go to a project
Toggle navigation Toggle navigation pinning
김승훈
b573f6f8 1 parent 6c3513df

update

Expand all
Inline Side-by-side
Showing 14 changed files with 577 additions and 0 deletions
var async = require('async');
var webdriver = require('selenium-webdriver');
var options = {desiredCapabilities: {browserName: 'chrome'}};
var JASON = require('jason')
const {Builder, By, Key, until} = require('selenium-webdriver');
var iconv = require('iconv-lite')
const fs = require('fs')
datalist = []
module.exports = {
search_onairanddate: function(driver,string,callback) {
let url1 = "https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query=" + string
driver
.get(url1).then(() => {
driver
.findElements(webdriver.By.className('brcs_detail'))
.then(broad_info => {
broad_info[0].findElement(webdriver.By.tagName('dl'))
.then(more_data => {
more_data.findElement(webdriver.By.tagName('dd'))
.then(one_line => {
one_line.findElement(webdriver.By.className("inline"))
.then(isbroad => {
isbroad.getText()
.then(isbroad_string => {
broadcasting=isbroad_string.substring(0,4)
isbroad = isbroad_string.substring(18,21)
br_date = isbroad_string.substring(23,24)
br_date = br_date + "요일"
br_time = isbroad_string.substring(26,34)
datalist.push(broadcasting,isbroad,br_date,br_time)
fs.writeFileSync("./log/"+string+"_br_isonair.txt", '\ufeff' + datalist, {encoding: 'utf8'});
for(var i =0; i<100; ) {
if() {
i++
}
}
callback()
})
})
})
})
})
})
driver.findElement(webdriver.By.className('brcs_thumb'))
.then(brcs_thumb => {
brcs_thumb.findElements(webdriver.By.tagName('img'))
.then( img => {
img[0].getAttribute('src')
.then(img_src => {
datalist.push(img_src)
})
})
})
}
}
\ No newline at end of file
var async = require('async');
var webdriver = require('selenium-webdriver');
var options = {desiredCapabilities: {browserName: 'chrome'}};
const {Builder, By, Key, until} = require('selenium-webdriver');
const fs = require('fs')
const screen = {
width: 640,
height: 480
};
var a=8000
var img_src_list = []
var text_list = []
var testStartRunTime = 0;
var testruntime = 0;
var time = 0;
module.exports = {
search_broadcasting :function(driver,url,name,callback) {
function testRunTimer() {
var today = new Date(); // 현재시간 얻기
var runTime = today.getTime(); // 밀리세컨드 ( 1970/01/01 부터 현재까지의 시간을 밀리세컨드로 나타냄 )
var rtn = 0;
if (testStartRunTime == 0) {
testStartRunTime = runTime;
} else {
rtn = (runTime - testStartRunTime) / 1000;
testStartRunTime = 0;
}
return rtn;
}
driver
.get("http://"+url)
.then(() => {
driver.findElement(webdriver.By.id('pagination_76'))
.then ( paginationBtn => {
paginationBtn.findElements(webdriver.By.className('bt_next'))
.then(Btn_next => {
driver.findElement(webdriver.By.id('listUI_76'))
.then(contentsUI => {
clickandget(function() {
Btn_next[0].click()
setTimeout(() => {
clickandget(function() {
Btn_next[0].click()
setTimeout(() => {
clickandget(function() {
Btn_next[0].click()
setTimeout(() => {
clickandget(function(){
Btn_next[0].click()
setTimeout(() => {
clickandget(function(){
fs.writeFileSync("./log/"+name+"_img_src_br.txt", '\ufeff' + img_src_list, {encoding: 'utf8'});
fs.writeFileSync("./log/"+name+"_title_br.txt", '\ufeff' + text_list, {encoding: 'utf8'});
setTimeout(() => {
callback()
}, 500);
})
}, testruntime*1000+800);
})
}, testruntime*1000+800);
},testruntime*1000+800)
}, testruntime*1000+800);
})
}, testruntime*1000+800);
})
function clickandget(_callback) {
testRunTimer()
Btn_next[0].isDisplayed().then(function(state) {
if(state) {
contentsUI.findElements(webdriver.By.tagName('li'))
.then(elems => {
elems.forEach(elem => {
elem.findElements(webdriver.By.tagName('img'))
.then(img => {
img[0].getAttribute('src')
.then(img_src => {
elem.findElements(webdriver.By.className('tit'))
.then (titles => {
titles.forEach(title => {
title.getText()
.then(text => {
text_list.push(text)
img_src_list.push(img_src)
if(text_list.length % 5 == 0) {
var testruntime = testRunTimer();
time = time+testruntime*1000
_callback()
}
})
})
})
})
})
})
})
}
else {
fs.writeFileSync("./log/"+name+"_img_src_br.txt", '\ufeff' + img_src_list, {encoding: 'utf8'});
fs.writeFileSync("./log/"+name+"_title_br.txt", '\ufeff' + text_list, {encoding: 'utf8'});
setTimeout(() => {
callback()
}, 500);
}
},function(err) {
fs.writeFileSync("./log/"+name+"_img_src_br.txt", '\ufeff' + img_src_list, {encoding: 'utf8'});
fs.writeFileSync("./log/"+name+"_title_br.txt", '\ufeff' + text_list, {encoding: 'utf8'});
setTimeout(() => {
callback()
}, 500);
})
}
})
})
})
})
function sleep (time) {
return new Promise((resolve) => setTimeout(resolve, time));
}
}
}
//var driver = new webdriver.Builder()
//.forBrowser('chrome')
//.withCapabilities(webdriver.Capabilities.chrome())
//.build();
//search_broadcasting(driver,"people.search.naver.com/search.naver?where=nexearch&sm=tab_ppn&query=유재석&os=94702&ie=utf8&key=PeopleService","유재석",function() {
// console.log("B")
//})
\ No newline at end of file
var async = require('async');
var webdriver = require('selenium-webdriver');
var chrome = require('selenium-webdriver/chrome')
var Options = new chrome.Options();
Options.addArguments('headless')
Options.addArguments('disable-gpu')
const fs = require('fs')
var check = 0
var testStartRunTime = 0;
var testruntime = 0;
const {Builder, By, Key, until} = require('selenium-webdriver');
var next_link_list = []
var src_list = []
var text_list = []
var return_list = []
module.exports = {
first_search :function (driver,string,callback) {
function testRunTimer() {
var today = new Date(); // 현재시간 얻기
var runTime = today.getTime(); // 밀리세컨드 ( 1970/01/01 부터 현재까지의 시간을 밀리세컨드로 나타냄 )
var rtn = 0;
if (testStartRunTime == 0) {
testStartRunTime = runTime;
} else {
rtn = (runTime - testStartRunTime) / 1000;
testStartRunTime = 0;
}
return rtn;
}
let url1 = "https://people.search.naver.com/search.naver?sm=tab_hty&where=nexearch&query="+string+"&ie=utf8&x=0&y=0"
driver
.get(url1).then(() => {
testRunTimer()
driver
.findElements(webdriver.By.className('result_profile'))
.then(profiles => {
profiles.forEach(profile => {
profile.findElements(webdriver.By.className('thmb'))
.then(links=> {
links.forEach(link => {
link.getAttribute('href')
.then(next_link => {
profile.findElements(webdriver.By.className('thmb_img'))
.then((imgs) => {
imgs.forEach(img => {
img.getAttribute('src')
.then(src => {
next_link_list.push(next_link)
src_list.push(src)
//text_list.push(text)
testruntime += testRunTimer()
setTimeout(function() {
check++;
if(check==1) {
//return_list.push(text_list)
setTimeout(function() {
fs.writeFileSync("./log/"+string+"_next_link.txt", '\ufeff' + next_link_list, {encoding: 'utf8'});
fs.writeFileSync("./log/"+string+"_img_src.txt", '\ufeff' + src_list, {encoding: 'utf8'});
callback()
},500)
}
},testruntime*1000+1000)
})
})
})
})
})
})
})
})
})
}
}
\ No newline at end of file
var webdriver = require('selenium-webdriver');
const {Builder, By, Key, until} = require('selenium-webdriver');
const fs = require('fs')
var arr = []
var result_arr = []
var temp = []
var i=0
var check = 0
module.exports = {
search_broadcasting_time : function(driver,broadcast,day,findtitle, callback) {
let url1 = "https://search.naver.com/search.naver?sm=top_hty&fbm=0&ie=utf8&query="+broadcast+"+"+day+"+편성표"
driver
.get(url1).then(() => {
driver
.findElements(webdriver.By.className('cont_inner type_day _scheduleArea'))
.then(contentsearch_section=> {
contentsearch_section[0].findElements(webdriver.By.tagName('tbody'))
.then(tbody => {
tbody[0].findElements(webdriver.By.tagName('tr'))
.then(trs => {
trs.forEach(tr => {
tr.getText()
.then(content => {
if(content.indexOf(findtitle) != -1) {
arr = content.toString().split('\n')
str(function() {
i++
if(i==2) {
fs.writeFileSync("./log/"+findtitle+"_time_detail.txt", '\ufeff' + result_arr, {encoding: 'utf8'});
callback()
}
})
function str(_callback) {
temp[i+0] = arr[0] + " " + arr[1].substr(0,2) + "분"
temp[i+1] = arr[1].substr(3)
result_arr.push(temp[i+0],temp[i+1])
if(arr.length == 3) {
temp[i+2] = arr[0] + " " + arr[2].substr(0,2) + "분"
temp[i+3] = arr[2].substr(3)
result_arr.push(temp[i+2],temp[i+3])
i++
}
setTimeout(() => {
_callback()
}, 500);
}
}
})
})
})
})
})
})
}
}
\ No newline at end of file
SBS ,방영중,일요일,오후 04:50
\ No newline at end of file
16시 50분,런닝맨 1부,17시 40분,런닝맨 2부
\ No newline at end of file
https://search.pstatic.net/common?type=a&size=60x76&quality=95&src=http://sstatic.naver.net/people/portrait/201808/20180828141741978.jpg,https://search.pstatic.net/common?type=a&size=60x76&quality=95&src=http://people.phinf.naver.net/20150328_223/1427473846485vwRXc_JPEG/chosun_400093103.jpg
\ No newline at end of file
https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/65/99/10/57_659910_poster_image_1506416659493.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/75/90/39/57_8759039_poster_image_1542880871064.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/75/91/55/57_8759155_poster_image_1540263433873.png,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/77/64/63/57_8776463_poster_image_1539219518133.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/66/33/78/57_8663378_poster_image_1534389778374.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/09/95/73/57_8099573_poster_image_1523413341476.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/44/21/84/57_6442184_poster_image_1514533716001.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/78/49/00/57_2784900_poster_image_1444978089345.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/44/71/54/57_2447154_poster_image_1433400789533.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/86/80/32/57_1868032_poster_image_1417592147530.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/67/49/81/57_674981_poster_image_1537158466551.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/0/0/164/164267.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/66/10/22/57_661022_poster_image_1457491725216.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/0/0/154/154271.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/0/0/154/154273.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/0/0/162/162685.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/0/0/159/159015.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/0/0/159/159705.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/0/0/160/160020.jpg,https://search.pstatic.net/common?type=mfullfill&size=96x138&quality=95&src=http://sstatic.naver.net/keypage/image/dss/57/65/91/74/57_659174_poster_image_1442911837885.jpg
\ No newline at end of file
https://people.search.naver.com/search.naver?where=nexearch&sm=tab_ppn&query=%EC%9C%A0%EC%9E%AC%EC%84%9D&os=94702&ie=utf8&key=PeopleService,https://people.search.naver.com/search.naver?where=nexearch&sm=tab_ppn&query=%EC%9C%A0%EC%9E%AC%EC%84%9D&os=362156&ie=utf8&key=PeopleService
\ No newline at end of file
무한도전,요즘애들,미추리 8-1000,해피투게더 4,유 퀴즈 온 더 블럭,범인은 바로 너!,투유 프로젝트 - 슈가맨 2,투유 프로젝트 - 슈가맨,동상이몽, 괜찮아 괜찮아!,나는 남자다,런닝맨,패밀리가 떴다 1,해피투게더 3,옛날 TV,하자 GO!,X맨 일요일이 좋다,강력추천 토요일,해피투게더 2,느낌표,공감토크쇼 놀러와
\ No newline at end of file
var async = require('async');
var webdriver = require('selenium-webdriver');
//var options = {desiredCapabilities: {browserName: 'chrome'}};
var chrome = require('selenium-webdriver/chrome')
var Options = new chrome.Options();
Options.addArguments('headless')
Options.addArguments('disable-gpu')
var test1 = require('./crawling_samename')
const fs = require('fs')
const test2 = require('./crawling_broadcasting')
const test3 = require('./crawling_br_onair_date')
const test4 = require('./crawling_time')
var util = require('util')
var EventEmitter = require('events').EventEmitter;
var next_link_Array = []
var img_src_Array = []
const {Builder, By, Key, until} = require('selenium-webdriver');
var driver = new webdriver.Builder()
.forBrowser('chrome')
.withCapabilities(webdriver.Capabilities.chrome())
.setChromeOptions(Options)
.build();
var driver_1 = new webdriver.Builder()
.forBrowser('chrome')
.withCapabilities(webdriver.Capabilities.chrome())
.setChromeOptions(Options)
.build()
var driver_2 = new webdriver.Builder()
.forBrowser('chrome')
.withCapabilities(webdriver.Capabilities.chrome())
.setChromeOptions(Options)
.build();
var driver_3 = new webdriver.Builder()
.forBrowser('chrome')
.withCapabilities(webdriver.Capabilities.chrome())
.setChromeOptions(Options)
.build();
var First_Search = function(name) {
var self=this;
this.on('input', function() {
if(fs.existsSync("./log/"+name+"_next_link.txt")) {
const next_link = fs.readFileSync("./log/"+name+"_next_link.txt");
const img_src = fs.readFileSync("./log/"+name+"_img_src.txt");
next_link_Array = next_link.toString().split(',');
img_src_Array = img_src.toString().split(',');
driver.quit()
console.log(next_link_Array)
console.log(img_src_Array)
setTimeout(function() {
br_search.emit('click')
},1000)
}
else {
test1.first_search(driver,name,function() {
const next_link = fs.readFileSync("./log/"+name+"_next_link.txt");
const img_src = fs.readFileSync("./log/"+name+"_img_src.txt");
next_link_Array = next_link.toString().split(',');
img_src_Array = img_src.toString().split(',');
br_search = new broadcasting_search(next_link_Array,"유재석",0)
console.log(next_link_Array)
console.log(img_src_Array)
driver.quit()
setTimeout(function() {
br_search.emit('click')
},1000)
})
}
})
}
util.inherits(First_Search,EventEmitter);
first_search1 = new First_Search("유재석")
first_search1.emit('input')
var img_src_br_list = []
var title_list = []
var broadcasting_search = function(next_link,name,num) {
var self=this;
this.on('click', function() {
if(fs.existsSync("./log/"+name+"_img_src_br.txt")) {
const img_src_br = fs.readFileSync("./log/"+name+"_img_src_br.txt");
img_src_br_list = img_src_br.toString().split(',');
const title_br = fs.readFileSync("./log/"+name+"_title_br.txt");
title_list = title_br.toString().split(',');
driver_1.quit()
console.log(img_src_br_list)
console.log(title_list)
is_onair.emit('click_1')
}
else {
test2.search_broadcasting(driver_1,next_link[num].substr(9),name,function() {
const img_src_br = fs.readFileSync("./log/"+name+"_img_src_br.txt");
img_src_br_list = img_src_br.toString().split(',');
const title_br = fs.readFileSync("./log/"+name+"_title_br.txt");
title_list = title_br.toString().split(',');
driver_1.quit()
console.log(img_src_br_list)
console.log(title_list)
is_onair.emit('click_1')
})
}
})
}
util.inherits(broadcasting_search,EventEmitter);
br_search = new broadcasting_search(next_link_Array,"유재석",0)
br_datalist = []
var isonair = function(num) {
var self = this
this.on('click_1',function() {
if(fs.existsSync("./log/"+title_list[num]+"_br_isonair.txt")) {
const img_src_br = fs.readFileSync("./log/"+title_list[num]+"_br_isonair.txt");
br_datalist = img_src_br.toString().split(',');
br_datalist[0] = br_datalist[0].trim()
console.log(br_datalist)
Search_detail_time.emit('click_2')
}
else {
test3.search_onairanddate(driver_2,title_list[num],function() {
const img_src_br = fs.readFileSync("./log/"+title_list[num]+"_br_isonair.txt");
br_datalist = img_src_br.toString().split(',');
br_datalist[0] = br_datalist[0].trim()
console.log(br_datalist)
Search_detail_time.emit('click_2')
})
}
})
}
util.inherits(isonair,EventEmitter);
is_onair = new isonair(11)
var detail_time_list = []
var search_detail_time = function(num) {
var self = this;
this.on('click_2',function() {
if(br_datalist[1] == "방영중") {
if(fs.existsSync("./log/"+title_list[num]+"_time_detail.txt")) {
const detail = fs.readFileSync("./log/"+title_list[num]+"_time_detail.txt");
detail_time_list = detail.toString().split(',');
console.log(detail_time_list)
}
else {
test4.search_broadcasting_time(driver_3,br_datalist[0],br_datalist[2],title_list[num],function(){
const detail = fs.readFileSync("./log/"+title_list[num]+"_time_detail.txt");
detail_time_list = detail.toString().split(',');
console.log(detail_time_list)
})
}
}
})
}
util.inherits(search_detail_time,EventEmitter);
Search_detail_time = new search_detail_time(11);
This diff is collapsed. Click to expand it.
{
"name": "cheerio",
"version": "1.0.0",
"description": "",
"main": "practice.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"async": "^2.6.1",
"events": "^3.0.0",
"fs": "0.0.1-security",
"iconv-lite": "^0.4.24",
"jason": "^2.0.0",
"json": "^9.0.6",
"selenium-webdriver": "^3.6.0",
"util": "^0.11.1",
"webdriverio": "^4.14.0"
}
}
var async = require('async');
var webdriver = require('selenium-webdriver');
//var options = {desiredCapabilities: {browserName: 'chrome'}};
var chrome = require('selenium-webdriver/chrome')
var Options = new chrome.Options();
Options.addArguments('headless')
Options.addArguments('disable-gpu')
var test1 = require('./crawling_samename')
const fs = require('fs')
var util = require('util')
var EventEmitter = require('events').EventEmitter;
var next_link_Array = []
var img_src_Array = []
const {Builder, By, Key, until} = require('selenium-webdriver');
var driver = new webdriver.Builder()
.forBrowser('chrome')
.withCapabilities(webdriver.Capabilities.chrome())
.setChromeOptions(Options)
.build();
var First_Search = function(name) {
var self=this;
this.on('input', function() {
if(fs.existsSync("./log/"+name+"_next_link.txt")) {
const next_link = fs.readFileSync("./log/"+name+"_next_link.txt");
const img_src = fs.readFileSync("./log/"+name+"_img_src.txt");
next_link_Array = next_link.toString().split(',');
img_src_Array = img_src.toString().split(',');
driver.quit()
console.log(next_link_Array)
console.log(img_src_Array)
setTimeout(function() {
br_search.emit('click')
},1000)
}
else {
test1.first_search(driver,name,function() {
const next_link = fs.readFileSync("./log/"+name+"_next_link.txt");
const img_src = fs.readFileSync("./log/"+name+"_img_src.txt");
next_link_Array = next_link.toString().split(',');
img_src_Array = img_src.toString().split(',');
br_search = new broadcasting_search(next_link_Array,"유재석",0)
console.log(next_link_Array)
console.log(img_src_Array)
driver.quit()
setTimeout(function() {
br_search.emit('click')
},1000)
})
}
})
}
util.inherits(First_Search,EventEmitter);
first_search1 = new First_Search("유재석")
first_search1.emit('input')