김대선 / Recruitment_Information_chatbot

Go to a project
Toggle navigation Toggle navigation pinning
김대선
8e371371 2 parents 86702b5f f186ea92

Merge branch 'develop' of http://khuhub.khu.ac.kr/2017103961/Recruitment_Informa… 

…tion_chatbot into release
Expand all
Inline Side-by-side
Showing 7 changed files with 304 additions and 77 deletions
1 -const express = require('express') 1 +var express = require('express');
2 +const request = require('request');
3 +const TARGET_URL = 'https://api.line.me/v2/bot/message/reply'
4 +const TOKEN = require('./config').TOKEN
5 +const fs = require('fs');
6 +const path = require('path');
7 +const HTTPS = require('https');
8 +const domain = require('./config').domain
9 +const sslport = 23023;
10 +const bodyParser = require('body-parser');
11 +
2 const schedule = require('node-schedule') 12 const schedule = require('node-schedule')
3 13
4 const data = require('./functions/dataFunctions') 14 const data = require('./functions/dataFunctions')
5 const find = require('./functions/findFunction') 15 const find = require('./functions/findFunction')
6 -const app = express();
7 -
8 -
9 16
10 // 0초 0분 0시 아무날 아무달 아무년 17 // 0초 0분 0시 아무날 아무달 아무년
11 -const saveData = schedule.scheduleJob('55 45 20 * * *', data.save) 18 +const saveData = schedule.scheduleJob('00 0 00 * * *', data.save)
12 19
13 -const server = app.listen(3000,()=>{ 20 +var app = express();
14 - const host = server.address().address 21 +app.use(bodyParser.json());
15 - const port = server.address().port 22 +app.post('/hook', function (req, res) {
16 - console.log("app listening at http://%s:%s", host, port) 23 + var eventObj = req.body.events[0];
17 -}) 24 +
25 + var source = eventObj.source;
26 + var message = eventObj.message;
27 + // request log
28 + console.log('======================', new Date() ,'======================');
29 + console.log('[queryString]', req.query)
30 + console.log('[request]', req.body);
31 + console.log('[request source] ', eventObj.source);
32 + console.log('[request message]', eventObj.message);
33 + console.log('[request postback]', eventObj.postback);
18 34
35 + let messageData
36 + const messageResult = []
37 + let string
38 + let start
39 + let finish
40 + let data
41 + let flag = true
42 + let button
43 + if(eventObj.type == "message"){
44 + start = 0;
45 + finish = start + 4
19 46
20 -app.get('/', async (req, res)=>{ 47 + button = {
48 + "type" : "flex",
49 + "altText" : "test FLEX",
50 + "contents" : {
51 + "type": "bubble",
52 + "body": {
53 + "type": "box",
54 + "layout" : "vertical",
55 + "contents" : [
56 + {
57 + "type": "button",
58 + "action": {
59 + "type":"postback",
60 + "label":"회사명으로 검색하기",
61 + "data": eventObj.message.text + "|||0|||" + "companyName"
62 + },
63 + "style": "primary",
64 + "color": "#ff9a9e"
65 + },
66 + {
67 + "type": "button",
68 + "action": {
69 + "type":"postback",
70 + "label":"태그로 검색하기",
71 + "data": eventObj.message.text + "|||0|||" + "tag"
72 + },
73 + "style": "primary",
74 + "color": "#fbc2eb"
75 + },
76 + {
77 + "type": "button",
78 + "action": {
79 + "type":"postback",
80 + "label":"제목으로 검색하기",
81 + "data": eventObj.message.text + "|||0|||" + "title"
82 + },
83 + "style": "primary",
84 + "color": "#8fd3f4"
85 + }
86 + ]
87 + }
88 + }
89 + }
90 + result = {
91 + url: TARGET_URL,
92 + headers: {
93 + 'Authorization': `Bearer ${TOKEN}`
94 + },
95 + json: {
96 + "replyToken":eventObj.replyToken,
97 + "messages": [button]
98 + }
99 + }
100 + request.post(result ,(error, response, body) => {
101 + console.log(body)
102 + });
103 +
104 + res.sendStatus(200);
105 + }
106 + else if(eventObj.type = "postback"){
107 + postbackData = eventObj.postback.data.split("|||")
108 + if(postbackData[2] == "companyName") messageData = find.byCompanyName(postbackData[0])
109 + else if(postbackData[2] == "tag") messageData = find.byTags(postbackData[0])
110 + else if(postbackData[2] == "title") messageData = find.byTitle(postbackData[0])
111 + if(messageData.length == 0){
112 + result = {
113 + url: TARGET_URL,
114 + headers: {
115 + 'Authorization': `Bearer ${TOKEN}`
116 + },
117 + json: {
118 + "replyToken":eventObj.replyToken,
119 + "messages": [{
120 + "type" : "text",
121 + "text" : "검색결과가 없습니다!"
122 + }]
123 + }
124 + }
125 + request.post(result ,(error, response, body) => {
126 + console.log(body)
127 + });
128 +
129 + res.sendStatus(200);
130 + }
131 + else{
132 + start = parseInt(eventObj.postback.data.split("|||")[1])
133 + finish = start + 4
134 + if (finish >= messageData.length){
135 + finish = messageData.length
136 + flag = false
137 + }
138 +
139 + button = {
140 + "type" : "flex",
141 + "altText" : "test FLEX",
142 + "contents" : {
143 + "type": "bubble",
144 + "body": {
145 + "type": "box",
146 + "layout" : "vertical",
147 + "contents" : [
148 + {
149 + "type": "button",
150 + "action": {
151 + "type":"postback",
152 + "label":"다음 보기",
153 + "data": postbackData[0] + " " + finish + " " + postbackData[2]
154 + },
155 + "style": "primary",
156 + "color": "#fbc2eb"
157 + }
158 + ]
159 + }
160 + }
161 + }
162 + for(start ; start < finish; start++){
163 + string = "제목 : " + messageData[start].title +"\n" + "회사명 : " + messageData[start].companyName + "\n" + "tags : " + messageData[start].tags.toString() +"\n" + "링크 : " + messageData[start].url + "\n"
164 + messageResult.push({
165 + "type" : "text",
166 + "text" : string
167 + })
168 + }
169 + if(flag){
170 + messageResult.push(button)
171 + result = {
172 + url: TARGET_URL,
173 + headers: {
174 + 'Authorization': `Bearer ${TOKEN}`
175 + },
176 + json: {
177 + "replyToken":eventObj.replyToken,
178 + "messages": messageResult
179 + }
180 + }
181 + }
182 + else{
183 + result = {
184 + url: TARGET_URL,
185 + headers: {
186 + 'Authorization': `Bearer ${TOKEN}`
187 + },
188 + json: {
189 + "replyToken":eventObj.replyToken,
190 + "messages": messageResult
191 + }
192 + }
193 + }
194 + request.post(result ,(error, response, body) => {
195 + console.log(body)
196 + });
197 +
198 + res.sendStatus(200);
199 + }
200 + }
201 +});
202 +try {
203 + const option = {
204 + ca: fs.readFileSync('/etc/letsencrypt/live/' + domain +'/fullchain.pem'),
205 + key: fs.readFileSync(path.resolve(process.cwd(), '/etc/letsencrypt/live/' + domain +'/privkey.pem'), 'utf8').toString(),
206 + cert: fs.readFileSync(path.resolve(process.cwd(), '/etc/letsencrypt/live/' + domain +'/cert.pem'), 'utf8').toString(),
207 + };
21 208
22 - console.log(find.byCompanyName('naver')[0], find.byTags('백엔드')[0], find.byTitle('백엔드')[0])
23 - res.send('helloworld')
24 -})
...\ No newline at end of file ...\ No newline at end of file
209 + HTTPS.createServer(option, app).listen(sslport, () => {
210 + console.log(`[HTTPS] Server is started on port ${sslport}`);
211 + });
212 +} catch (error) {
213 + console.log('[HTTPS] HTTPS 오류가 발생하였습니다. HTTPS 서버는 실행되지 않습니다.');
214 + console.log(error);
215 +}
......
1 const fs = require('fs') 1 const fs = require('fs')
2 const functions = require('./function') 2 const functions = require('./function')
3 3
4 -const read = () =>{ 4 +let status = false
5 - const data = []
6 - let title = ""
7 - let tags = []
8 - let url = ""
9 - let companyName = ""
10 - const today = new Date()
11 - const string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, 'utf-8', 'r')
12 - const stringArray = string.split('\n')
13 - const size = stringArray.length
14 - for(let i = 0 ; i < size; i++){
15 - if( i % 4 == 0){
16 - title = stringArray[i].replace("title : ", "")
17 - }
18 - else if( i % 4 == 1){
19 - tags = stringArray[i].replace("tags : ", "").split(",")
20 - }
21 - else if( i % 4 == 2){
22 - url = stringArray[i].replace("url : ", "")
23 - }
24 - else if ( i % 4 == 3){
25 - companyName = stringArray[i].replace("companyName : ", "")
26 - data.push({
27 - title : title,
28 - tags : tags,
29 - url : url,
30 - companyName : companyName
31 - })
32 - }
33 - }
34 - return data
35 -}
36 5
37 const save = async()=> { 6 const save = async()=> {
7 + if(status){
8 + return
9 + }
10 + else if(!status){
11 + status = true
12 + }
38 try { 13 try {
39 let string = "" 14 let string = ""
40 let data = await functions.getKakaoData() 15 let data = await functions.getKakaoData()
...@@ -49,7 +24,6 @@ const save = async()=> { ...@@ -49,7 +24,6 @@ const save = async()=> {
49 temp = "companyName : " + i.companyName 24 temp = "companyName : " + i.companyName
50 string = string + temp + "\n" 25 string = string + temp + "\n"
51 } 26 }
52 -
53 data = await functions.getNaverFunction() 27 data = await functions.getNaverFunction()
54 temp = "" 28 temp = ""
55 for(let i of data){ 29 for(let i of data){
...@@ -62,7 +36,10 @@ const save = async()=> { ...@@ -62,7 +36,10 @@ const save = async()=> {
62 temp = "companyName : " + i.companyName 36 temp = "companyName : " + i.companyName
63 string = string + temp + "\n" 37 string = string + temp + "\n"
64 } 38 }
65 - 39 +<<<<<<< HEAD
40 +
41 +=======
42 +>>>>>>> 694647de3d94a82e2485ce202270c0702aaed6f7
66 data = await functions.getProgrammersFunction() 43 data = await functions.getProgrammersFunction()
67 temp = "" 44 temp = ""
68 for(let i of data){ 45 for(let i of data){
...@@ -75,23 +52,79 @@ const save = async()=> { ...@@ -75,23 +52,79 @@ const save = async()=> {
75 temp = "companyName : " + i.companyName 52 temp = "companyName : " + i.companyName
76 string = string + temp + "\n" 53 string = string + temp + "\n"
77 } 54 }
55 +
78 const today = new Date() 56 const today = new Date()
79 fs.writeFile(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, string, 'utf-8', (err)=>{ 57 fs.writeFile(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, string, 'utf-8', (err)=>{
58 + status = false
80 if(err){ 59 if(err){
81 console.log("파일저장시에 오류") 60 console.log("파일저장시에 오류")
82 - console.log(err)
83 - save()
84 } 61 }
85 else console.log("저장완료!") 62 else console.log("저장완료!")
86 }) 63 })
87 } catch (error) { 64 } catch (error) {
88 - console.log("데이터 가져오는 과정에서 오류") 65 + status = false
89 - console.log(error) 66 + console.log("파일저장시에 오류")
67 + }
68 +}
69 +
70 +const read = () =>{
71 + const data = []
72 + let title = ""
73 + let tags = []
74 + let url = ""
75 + let companyName = ""
76 + // const today = new Date()
77 + const today = new Date()
78 + let string
79 + try {
80 + string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, 'utf-8', 'r')
81 + } catch (error) {
90 save() 82 save()
83 + if(today.getDate()-1 == 0){
84 + if([1, 3, 5, 7, 8, 10, 12].includes(today.getMonth() - 1)){
85 + string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth() - 1}.${31}`, 'utf-8', 'r')
86 + }
87 + else if([4, 6, 9, 11].includes(today.getMonth() - 1)){
88 + string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${30}`, 'utf-8', 'r')
89 + }
90 + else if (2 == today.getMonth() - 1){
91 + string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${28}`, 'utf-8', 'r')
92 + }
93 + else if (0 == today.getMonth() - 1){
94 + string = fs.readFileSync(`./datas/${today.getFullYear() - 1}.${12}.${31}`, 'utf-8', 'r')
95 + }
96 + }
97 + else{
98 + string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()-1}`, 'utf-8', 'r')
99 + }
91 } 100 }
101 + return string
102 + const stringArray = string.split('\n')
103 + const size = stringArray.length
104 + for(let i = 0 ; i < size; i++){
105 + if( i % 4 == 0){
106 + title = stringArray[i].replace("title : ", "")
107 + }
108 + else if( i % 4 == 1){
109 + tags = stringArray[i].replace("tags : ", "").split(",")
110 + }
111 + else if( i % 4 == 2){
112 + url = stringArray[i].replace("url : ", "")
113 + }
114 + else if ( i % 4 == 3){
115 + companyName = stringArray[i].replace("companyName : ", "")
116 + data.push({
117 + title : title,
118 + tags : tags,
119 + url : url,
120 + companyName : companyName
121 + })
122 + }
123 + }
124 + return data
92 } 125 }
93 126
94 module.exports = { 127 module.exports = {
95 save : save, 128 save : save,
96 read : read 129 read : read
97 -}
...\ No newline at end of file ...\ No newline at end of file
130 +}
......
...@@ -32,24 +32,19 @@ const makeObject = (array)=>{ ...@@ -32,24 +32,19 @@ const makeObject = (array)=>{
32 return result 32 return result
33 } 33 }
34 34
35 -const moveNextPage = async (page)=>{
36 -
37 - await page.click('#mArticle > div > div.paging_list > span > a:nth-child(10) > span > span').catch((error)=>{
38 - })
39 - await page.waitForTimeout(1000)
40 - return await page.content()
41 -}
42 -
43 const getData = async ()=>{ 35 const getData = async ()=>{
44 const browser = await puppeteer.launch(); 36 const browser = await puppeteer.launch();
45 const page = await browser.newPage(); 37 const page = await browser.newPage();
38 + page.setDefaultNavigationTimeout(0)
46 39
47 let result = [] 40 let result = []
48 let temp = "" 41 let temp = ""
49 - 42 + let count = 1
50 await page.goto('https://careers.kakao.com/jobs') 43 await page.goto('https://careers.kakao.com/jobs')
51 - let content = await page.content() 44 + let content = ""
52 while(true){ 45 while(true){
46 + await page.goto(`https://careers.kakao.com/jobs?page=${count}`)
47 + content = await page.content()
53 if(temp == content){ 48 if(temp == content){
54 break; 49 break;
55 } 50 }
...@@ -66,12 +61,12 @@ const getData = async ()=>{ ...@@ -66,12 +61,12 @@ const getData = async ()=>{
66 resArr.pop() 61 resArr.pop()
67 result = result.concat(await makeObject(resArr)) 62 result = result.concat(await makeObject(resArr))
68 temp = content 63 temp = content
69 - content = await moveNextPage(page) 64 + count = count + 1
70 } 65 }
66 + console.log("kakao : ", result.length)
71 return result 67 return result
72 } 68 }
73 69
74 module.exports = { 70 module.exports = {
75 getData : getData 71 getData : getData
76 } 72 }
77 -
......
...@@ -40,19 +40,25 @@ const makeObject = (array)=>{ ...@@ -40,19 +40,25 @@ const makeObject = (array)=>{
40 const getData = async ()=>{ 40 const getData = async ()=>{
41 const browser = await puppeteer.launch(); 41 const browser = await puppeteer.launch();
42 const page = await browser.newPage(); 42 const page = await browser.newPage();
43 + page.setDefaultNavigationTimeout(0)
43 44
44 await page.goto('https://recruit.navercorp.com/naver/job/list/developer') 45 await page.goto('https://recruit.navercorp.com/naver/job/list/developer')
45 let content = await page.content() 46 let content = await page.content()
46 - let temp = null; 47 + let temp = "";
47 - while(true){ 48 + let Flag = true
48 - if (temp == content) break; 49 + while(Flag){
50 + if (temp == content){
51 + Flag = false
52 + break;
53 + }
49 else{ 54 else{
50 - temp = await page.content() 55 + temp = content
51 await page.click('#moreDiv > button').catch((error)=>{ 56 await page.click('#moreDiv > button').catch((error)=>{
52 - 57 + Flag = false
53 }) 58 })
54 - await page.waitForTimeout(200) 59 + await page.waitForTimeout(2000)
55 content = await page.content() 60 content = await page.content()
61 +
56 } 62 }
57 } 63 }
58 let $ = cheerio.load(content, {decodeEntities: true}) 64 let $ = cheerio.load(content, {decodeEntities: true})
...@@ -66,9 +72,10 @@ const getData = async ()=>{ ...@@ -66,9 +72,10 @@ const getData = async ()=>{
66 }) 72 })
67 resArr = result.split('</li><li>') 73 resArr = result.split('</li><li>')
68 result = makeObject(resArr) 74 result = makeObject(resArr)
75 + console.log("naver : ", result.length)
69 return result; 76 return result;
70 } 77 }
71 78
72 module.exports = { 79 module.exports = {
73 getData : getData 80 getData : getData
74 -}
...\ No newline at end of file ...\ No newline at end of file
81 +}
......
...@@ -43,7 +43,8 @@ const makeObject = (array)=>{ ...@@ -43,7 +43,8 @@ const makeObject = (array)=>{
43 const getData = async ()=>{ 43 const getData = async ()=>{
44 const browser = await puppeteer.launch(); 44 const browser = await puppeteer.launch();
45 const page = await browser.newPage(); 45 const page = await browser.newPage();
46 - 46 + page.setDefaultNavigationTimeout(0)
47 +
47 let result = [] 48 let result = []
48 let temp = "" 49 let temp = ""
49 let count = 1; 50 let count = 1;
...@@ -56,12 +57,10 @@ const getData = async ()=>{ ...@@ -56,12 +57,10 @@ const getData = async ()=>{
56 decodeEntities : true 57 decodeEntities : true
57 } 58 }
58 }).replace(/(<([^>]+)>)*(\\t)?/gi, "") 59 }).replace(/(<([^>]+)>)*(\\t)?/gi, "")
59 - console.log(final)
60 while(true){ 60 while(true){
61 await page.goto(`https://programmers.co.kr/job?page=${count}`) 61 await page.goto(`https://programmers.co.kr/job?page=${count}`)
62 content = await page.content() 62 content = await page.content()
63 if(final < count){ 63 if(final < count){
64 - console.log("finish", result.length)
65 break; 64 break;
66 } 65 }
67 $ = cheerio.load(content, {decodeEntities: true}) 66 $ = cheerio.load(content, {decodeEntities: true})
...@@ -76,7 +75,6 @@ const getData = async ()=>{ ...@@ -76,7 +75,6 @@ const getData = async ()=>{
76 } 75 }
77 }) 76 })
78 if(item ==''){ 77 if(item ==''){
79 - console.log("break!!!!")
80 break; 78 break;
81 } 79 }
82 item = item.split("</div>`") 80 item = item.split("</div>`")
...@@ -85,8 +83,8 @@ const getData = async ()=>{ ...@@ -85,8 +83,8 @@ const getData = async ()=>{
85 } 83 }
86 result = result.concat(await makeObject(resArr)) 84 result = result.concat(await makeObject(resArr))
87 count = count + 1 85 count = count + 1
88 - resArr = []
89 } 86 }
87 + console.log("kakao : ", result.length)
90 return result 88 return result
91 } 89 }
92 90
......
This diff is collapsed. Click to expand it.
...@@ -14,10 +14,13 @@ ...@@ -14,10 +14,13 @@
14 "author": "김대선", 14 "author": "김대선",
15 "license": "ISC", 15 "license": "ISC",
16 "dependencies": { 16 "dependencies": {
17 + "body-parser": "^1.19.0",
17 "cheerio": "^1.0.0-rc.9", 18 "cheerio": "^1.0.0-rc.9",
18 "express": "^4.17.1", 19 "express": "^4.17.1",
19 "node-schedule": "^2.0.0", 20 "node-schedule": "^2.0.0",
20 "puppeteer": "^9.1.1", 21 "puppeteer": "^9.1.1",
22 + "puppeteer-core": "^10.0.0",
23 + "request": "^2.88.2",
21 "sanitize-html": "^2.3.3" 24 "sanitize-html": "^2.3.3"
22 } 25 }
23 } 26 }
......