김대선 / Recruitment_Information_chatbot

Go to a project
Toggle navigation Toggle navigation pinning
김대선
8e371371 2 parents 86702b5f f186ea92

Merge branch 'develop' of http://khuhub.khu.ac.kr/2017103961/Recruitment_Informa… 

…tion_chatbot into release
Expand all
Inline Side-by-side
Showing 7 changed files with 304 additions and 77 deletions
const express = require('express')
var express = require('express');
const request = require('request');
const TARGET_URL = 'https://api.line.me/v2/bot/message/reply'
const TOKEN = require('./config').TOKEN
const fs = require('fs');
const path = require('path');
const HTTPS = require('https');
const domain = require('./config').domain
const sslport = 23023;
const bodyParser = require('body-parser');
const schedule = require('node-schedule')
const data = require('./functions/dataFunctions')
const find = require('./functions/findFunction')
const app = express();
// 0초 0분 0시 아무날 아무달 아무년
const saveData = schedule.scheduleJob('55 45 20 * * *', data.save)
const saveData = schedule.scheduleJob('00 0 00 * * *', data.save)
const server = app.listen(3000,()=>{
const host = server.address().address
const port = server.address().port
console.log("app listening at http://%s:%s", host, port)
})
var app = express();
app.use(bodyParser.json());
app.post('/hook', function (req, res) {
var eventObj = req.body.events[0];
var source = eventObj.source;
var message = eventObj.message;
// request log
console.log('======================', new Date() ,'======================');
console.log('[queryString]', req.query)
console.log('[request]', req.body);
console.log('[request source] ', eventObj.source);
console.log('[request message]', eventObj.message);
console.log('[request postback]', eventObj.postback);
let messageData
const messageResult = []
let string
let start
let finish
let data
let flag = true
let button
if(eventObj.type == "message"){
start = 0;
finish = start + 4
app.get('/', async (req, res)=>{
button = {
"type" : "flex",
"altText" : "test FLEX",
"contents" : {
"type": "bubble",
"body": {
"type": "box",
"layout" : "vertical",
"contents" : [
{
"type": "button",
"action": {
"type":"postback",
"label":"회사명으로 검색하기",
"data": eventObj.message.text + "|||0|||" + "companyName"
},
"style": "primary",
"color": "#ff9a9e"
},
{
"type": "button",
"action": {
"type":"postback",
"label":"태그로 검색하기",
"data": eventObj.message.text + "|||0|||" + "tag"
},
"style": "primary",
"color": "#fbc2eb"
},
{
"type": "button",
"action": {
"type":"postback",
"label":"제목으로 검색하기",
"data": eventObj.message.text + "|||0|||" + "title"
},
"style": "primary",
"color": "#8fd3f4"
}
]
}
}
}
result = {
url: TARGET_URL,
headers: {
'Authorization': `Bearer ${TOKEN}`
},
json: {
"replyToken":eventObj.replyToken,
"messages": [button]
}
}
request.post(result ,(error, response, body) => {
console.log(body)
});
res.sendStatus(200);
}
else if(eventObj.type = "postback"){
postbackData = eventObj.postback.data.split("|||")
if(postbackData[2] == "companyName") messageData = find.byCompanyName(postbackData[0])
else if(postbackData[2] == "tag") messageData = find.byTags(postbackData[0])
else if(postbackData[2] == "title") messageData = find.byTitle(postbackData[0])
if(messageData.length == 0){
result = {
url: TARGET_URL,
headers: {
'Authorization': `Bearer ${TOKEN}`
},
json: {
"replyToken":eventObj.replyToken,
"messages": [{
"type" : "text",
"text" : "검색결과가 없습니다!"
}]
}
}
request.post(result ,(error, response, body) => {
console.log(body)
});
res.sendStatus(200);
}
else{
start = parseInt(eventObj.postback.data.split("|||")[1])
finish = start + 4
if (finish >= messageData.length){
finish = messageData.length
flag = false
}
button = {
"type" : "flex",
"altText" : "test FLEX",
"contents" : {
"type": "bubble",
"body": {
"type": "box",
"layout" : "vertical",
"contents" : [
{
"type": "button",
"action": {
"type":"postback",
"label":"다음 보기",
"data": postbackData[0] + " " + finish + " " + postbackData[2]
},
"style": "primary",
"color": "#fbc2eb"
}
]
}
}
}
for(start ; start < finish; start++){
string = "제목 : " + messageData[start].title +"\n" + "회사명 : " + messageData[start].companyName + "\n" + "tags : " + messageData[start].tags.toString() +"\n" + "링크 : " + messageData[start].url + "\n"
messageResult.push({
"type" : "text",
"text" : string
})
}
if(flag){
messageResult.push(button)
result = {
url: TARGET_URL,
headers: {
'Authorization': `Bearer ${TOKEN}`
},
json: {
"replyToken":eventObj.replyToken,
"messages": messageResult
}
}
}
else{
result = {
url: TARGET_URL,
headers: {
'Authorization': `Bearer ${TOKEN}`
},
json: {
"replyToken":eventObj.replyToken,
"messages": messageResult
}
}
}
request.post(result ,(error, response, body) => {
console.log(body)
});
res.sendStatus(200);
}
}
});
try {
const option = {
ca: fs.readFileSync('/etc/letsencrypt/live/' + domain +'/fullchain.pem'),
key: fs.readFileSync(path.resolve(process.cwd(), '/etc/letsencrypt/live/' + domain +'/privkey.pem'), 'utf8').toString(),
cert: fs.readFileSync(path.resolve(process.cwd(), '/etc/letsencrypt/live/' + domain +'/cert.pem'), 'utf8').toString(),
};
console.log(find.byCompanyName('naver')[0], find.byTags('백엔드')[0], find.byTitle('백엔드')[0])
res.send('helloworld')
})
\ No newline at end of file
HTTPS.createServer(option, app).listen(sslport, () => {
console.log(`[HTTPS] Server is started on port ${sslport}`);
});
} catch (error) {
console.log('[HTTPS] HTTPS 오류가 발생하였습니다. HTTPS 서버는 실행되지 않습니다.');
console.log(error);
}
......
const fs = require('fs')
const functions = require('./function')
const read = () =>{
const data = []
let title = ""
let tags = []
let url = ""
let companyName = ""
const today = new Date()
const string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, 'utf-8', 'r')
const stringArray = string.split('\n')
const size = stringArray.length
for(let i = 0 ; i < size; i++){
if( i % 4 == 0){
title = stringArray[i].replace("title : ", "")
}
else if( i % 4 == 1){
tags = stringArray[i].replace("tags : ", "").split(",")
}
else if( i % 4 == 2){
url = stringArray[i].replace("url : ", "")
}
else if ( i % 4 == 3){
companyName = stringArray[i].replace("companyName : ", "")
data.push({
title : title,
tags : tags,
url : url,
companyName : companyName
})
}
}
return data
}
let status = false
const save = async()=> {
if(status){
return
}
else if(!status){
status = true
}
try {
let string = ""
let data = await functions.getKakaoData()
......@@ -49,7 +24,6 @@ const save = async()=> {
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
data = await functions.getNaverFunction()
temp = ""
for(let i of data){
......@@ -62,7 +36,10 @@ const save = async()=> {
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
<<<<<<< HEAD
=======
>>>>>>> 694647de3d94a82e2485ce202270c0702aaed6f7
data = await functions.getProgrammersFunction()
temp = ""
for(let i of data){
......@@ -75,23 +52,79 @@ const save = async()=> {
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
const today = new Date()
fs.writeFile(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, string, 'utf-8', (err)=>{
status = false
if(err){
console.log("파일저장시에 오류")
console.log(err)
save()
}
else console.log("저장완료!")
})
} catch (error) {
console.log("데이터 가져오는 과정에서 오류")
console.log(error)
status = false
console.log("파일저장시에 오류")
}
}
const read = () =>{
const data = []
let title = ""
let tags = []
let url = ""
let companyName = ""
// const today = new Date()
const today = new Date()
let string
try {
string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, 'utf-8', 'r')
} catch (error) {
save()
if(today.getDate()-1 == 0){
if([1, 3, 5, 7, 8, 10, 12].includes(today.getMonth() - 1)){
string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth() - 1}.${31}`, 'utf-8', 'r')
}
else if([4, 6, 9, 11].includes(today.getMonth() - 1)){
string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${30}`, 'utf-8', 'r')
}
else if (2 == today.getMonth() - 1){
string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${28}`, 'utf-8', 'r')
}
else if (0 == today.getMonth() - 1){
string = fs.readFileSync(`./datas/${today.getFullYear() - 1}.${12}.${31}`, 'utf-8', 'r')
}
}
else{
string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()-1}`, 'utf-8', 'r')
}
}
return string
const stringArray = string.split('\n')
const size = stringArray.length
for(let i = 0 ; i < size; i++){
if( i % 4 == 0){
title = stringArray[i].replace("title : ", "")
}
else if( i % 4 == 1){
tags = stringArray[i].replace("tags : ", "").split(",")
}
else if( i % 4 == 2){
url = stringArray[i].replace("url : ", "")
}
else if ( i % 4 == 3){
companyName = stringArray[i].replace("companyName : ", "")
data.push({
title : title,
tags : tags,
url : url,
companyName : companyName
})
}
}
return data
}
module.exports = {
save : save,
read : read
}
\ No newline at end of file
}
......
......@@ -32,24 +32,19 @@ const makeObject = (array)=>{
return result
}
const moveNextPage = async (page)=>{
await page.click('#mArticle > div > div.paging_list > span > a:nth-child(10) > span > span').catch((error)=>{
})
await page.waitForTimeout(1000)
return await page.content()
}
const getData = async ()=>{
const browser = await puppeteer.launch();
const page = await browser.newPage();
page.setDefaultNavigationTimeout(0)
let result = []
let temp = ""
let count = 1
await page.goto('https://careers.kakao.com/jobs')
let content = await page.content()
let content = ""
while(true){
await page.goto(`https://careers.kakao.com/jobs?page=${count}`)
content = await page.content()
if(temp == content){
break;
}
......@@ -66,12 +61,12 @@ const getData = async ()=>{
resArr.pop()
result = result.concat(await makeObject(resArr))
temp = content
content = await moveNextPage(page)
count = count + 1
}
console.log("kakao : ", result.length)
return result
}
module.exports = {
getData : getData
}
......
......@@ -40,19 +40,25 @@ const makeObject = (array)=>{
const getData = async ()=>{
const browser = await puppeteer.launch();
const page = await browser.newPage();
page.setDefaultNavigationTimeout(0)
await page.goto('https://recruit.navercorp.com/naver/job/list/developer')
let content = await page.content()
let temp = null;
while(true){
if (temp == content) break;
let temp = "";
let Flag = true
while(Flag){
if (temp == content){
Flag = false
break;
}
else{
temp = await page.content()
temp = content
await page.click('#moreDiv > button').catch((error)=>{
Flag = false
})
await page.waitForTimeout(200)
await page.waitForTimeout(2000)
content = await page.content()
}
}
let $ = cheerio.load(content, {decodeEntities: true})
......@@ -66,9 +72,10 @@ const getData = async ()=>{
})
resArr = result.split('</li><li>')
result = makeObject(resArr)
console.log("naver : ", result.length)
return result;
}
module.exports = {
getData : getData
}
\ No newline at end of file
}
......
......@@ -43,7 +43,8 @@ const makeObject = (array)=>{
const getData = async ()=>{
const browser = await puppeteer.launch();
const page = await browser.newPage();
page.setDefaultNavigationTimeout(0)
let result = []
let temp = ""
let count = 1;
......@@ -56,12 +57,10 @@ const getData = async ()=>{
decodeEntities : true
}
}).replace(/(<([^>]+)>)*(\\t)?/gi, "")
console.log(final)
while(true){
await page.goto(`https://programmers.co.kr/job?page=${count}`)
content = await page.content()
if(final < count){
console.log("finish", result.length)
break;
}
$ = cheerio.load(content, {decodeEntities: true})
......@@ -76,7 +75,6 @@ const getData = async ()=>{
}
})
if(item ==''){
console.log("break!!!!")
break;
}
item = item.split("</div>`")
......@@ -85,8 +83,8 @@ const getData = async ()=>{
}
result = result.concat(await makeObject(resArr))
count = count + 1
resArr = []
}
console.log("kakao : ", result.length)
return result
}
......
This diff is collapsed. Click to expand it.
......@@ -14,10 +14,13 @@
"author": "김대선",
"license": "ISC",
"dependencies": {
"body-parser": "^1.19.0",
"cheerio": "^1.0.0-rc.9",
"express": "^4.17.1",
"node-schedule": "^2.0.0",
"puppeteer": "^9.1.1",
"puppeteer-core": "^10.0.0",
"request": "^2.88.2",
"sanitize-html": "^2.3.3"
}
}
......