김대선 / Recruitment_Information_chatbot

Go to a project
Toggle navigation Toggle navigation pinning
김대선
51f76856 1 parent 19ab390d

데이터 저장, 읽기 분리

Inline Side-by-side
Showing 6 changed files with 113 additions and 92 deletions
1 const express = require('express') 1 const express = require('express')
2 const schedule = require('node-schedule') 2 const schedule = require('node-schedule')
3 -const fs = require('fs')
4 -const functions = require('./function')
5 -
6 -const app = express();
7 3
4 +const dataFunctions = require('./functions/dataFunctions')
8 5
9 -const readData = async () =>{ 6 +const app = express();
10 - const data = []
11 - let title = ""
12 - let tags = []
13 - let url = ""
14 - const today = new Date()
15 - const string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, 'utf-8', 'r')
16 - const stringArray = string.split('\n')
17 - const size = stringArray.length
18 - for(let i = 0 ; i < size; i++){
19 - if( i % 3 == 0){
20 - title = stringArray[i].replace("title : ", "")
21 - }
22 - else if( i % 3 == 1){
23 - tags = stringArray[i].replace("tags : ", "").split(",")
24 - }
25 - else if ( i % 3 == 2){
26 - url = stringArray[i].replace("url : ", "")
27 - data.push({
28 - title : title,
29 - tags : tags,
30 - url : url
31 - })
32 - }
33 - }
34 - console.log(data)
35 -}
36 7
37 -const save = async()=> {
38 - try {
39 - let string = ""
40 - let data = await functions.getKakaoData()
41 - let temp = ""
42 - for(let i of data){
43 - temp = "title : " + i.title
44 - string = string + temp + "\n"
45 - temp = "tags : " + i.tags.toString()
46 - string = string + temp + "\n"
47 - temp = "url : " + i.url
48 - string = string + temp + "\n"
49 - }
50 8
51 - data = await functions.getNaverFunction()
52 - temp = ""
53 - for(let i of data){
54 - temp = "title : " + i.title
55 - string = string + temp + "\n"
56 - if(!!i.tags){
57 - temp = "tags : " + i.tags.toString()
58 - string = string + temp + "\n"
59 - }
60 - temp = "url : " + i.url
61 - string = string + temp + "\n"
62 - }
63 9
64 - data = await functions.getProgrammersFunction()
65 - temp = ""
66 - for(let i of data){
67 - temp = "title : " + i.title
68 - string = string + temp + "\n"
69 - temp = "tags : " + i.tags.toString()
70 - string = string + temp + "\n"
71 - temp = "url : " + i.url
72 - string = string + temp + "\n"
73 - }
74 - const today = new Date()
75 - fs.writeFile(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, string, 'utf-8', (err)=>{
76 - if(err){
77 - save()
78 - }
79 - else console.log("저장완료!")
80 - })
81 - } catch (error) {
82 - console.log(error)
83 - save()
84 - }
85 -}
86 // 0초 0분 0시 아무날 아무달 아무년 10 // 0초 0분 0시 아무날 아무달 아무년
87 -const saveData = schedule.scheduleJob('0 0 0 * * *', save) 11 +const saveData = schedule.scheduleJob('55 30 16 * * *', dataFunctions.save)
88 12
89 const server = app.listen(3000,()=>{ 13 const server = app.listen(3000,()=>{
90 const host = server.address().address 14 const host = server.address().address
...@@ -96,5 +20,5 @@ const server = app.listen(3000,()=>{ ...@@ -96,5 +20,5 @@ const server = app.listen(3000,()=>{
96 app.get('/', async (req, res)=>{ 20 app.get('/', async (req, res)=>{
97 21
98 22
99 - res.send(string) 23 + res.send("hello world")
100 }) 24 })
...\ No newline at end of file ...\ No newline at end of file
......
1 +const fs = require('fs')
2 +const functions = require('./function')
3 +
4 +const read = async () =>{
5 + const data = []
6 + let title = ""
7 + let tags = []
8 + let url = ""
9 + let companyName = ""
10 + const today = new Date()
11 + const string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, 'utf-8', 'r')
12 + const stringArray = string.split('\n')
13 + const size = stringArray.length
14 + for(let i = 0 ; i < size; i++){
15 + if( i % 4 == 0){
16 + title = stringArray[i].replace("title : ", "")
17 + }
18 + else if( i % 4 == 1){
19 + tags = stringArray[i].replace("tags : ", "").split(",")
20 + }
21 + else if( i % 4 == 2){
22 + tags = stringArray[i].replace("url : ", "")
23 + }
24 + else if ( i % 4 == 3){
25 + companyName = stringArray[i].replace("company : ", "")
26 + data.push({
27 + title : title,
28 + tags : tags,
29 + url : url,
30 + companyName : companyName
31 + })
32 + }
33 + }
34 + console.log(data)
35 +}
36 +
37 +const save = async()=> {
38 + try {
39 + let string = ""
40 + let data = await functions.getKakaoData()
41 + let temp = ""
42 + for(let i of data){
43 + temp = "title : " + i.title
44 + string = string + temp + "\n"
45 + temp = "tags : " + i.tags.toString()
46 + string = string + temp + "\n"
47 + temp = "url : " + i.url
48 + string = string + temp + "\n"
49 + temp = "companyName : " + i.companyName
50 + string = string + temp + "\n"
51 + }
52 +
53 + data = await functions.getNaverFunction()
54 + temp = ""
55 + for(let i of data){
56 + temp = "title : " + i.title
57 + string = string + temp + "\n"
58 + temp = "tags : " + i.tags.toString()
59 + string = string + temp + "\n"
60 + temp = "url : " + i.url
61 + string = string + temp + "\n"
62 + temp = "companyName : " + i.companyName
63 + string = string + temp + "\n"
64 + }
65 +
66 + data = await functions.getProgrammersFunction()
67 + temp = ""
68 + for(let i of data){
69 + temp = "title : " + i.title
70 + string = string + temp + "\n"
71 + temp = "tags : " + i.tags.toString()
72 + string = string + temp + "\n"
73 + temp = "url : " + i.url
74 + string = string + temp + "\n"
75 + temp = "companyName : " + i.companyName
76 + string = string + temp + "\n"
77 + }
78 + const today = new Date()
79 + fs.writeFile(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, string, 'utf-8', (err)=>{
80 + if(err){
81 + console.log(err)
82 + save()
83 + }
84 + else console.log("저장완료!")
85 + })
86 + } catch (error) {
87 + console.log(error)
88 + save()
89 + }
90 +}
91 +
92 +module.exports = {
93 + save : save,
94 + read : read
95 +}
...\ No newline at end of file ...\ No newline at end of file
1 -const kakaoFunction = require("./functions/kakaoFunction"); 1 +const kakaoFunction = require("./kakaoFunction");
2 -const naverFunction = require("./functions/naverFunction"); 2 +const naverFunction = require("./naverFunction");
3 -const programmersFunction = require("./functions/programmersFunction") 3 +const programmersFunction = require("./programmersFunction")
4 4
5 5
6 module.exports = { 6 module.exports = {
......
...@@ -20,7 +20,8 @@ const makeObject = (array)=>{ ...@@ -20,7 +20,8 @@ const makeObject = (array)=>{
20 result.push({ 20 result.push({
21 title : tempData[0], 21 title : tempData[0],
22 tags : [], 22 tags : [],
23 - url : tempData[tempData.length - 1] 23 + url : tempData[tempData.length - 1],
24 + companyName : "kakao"
24 }) 25 })
25 for(let j of tempData){ 26 for(let j of tempData){
26 if(j[0] == '#'){ 27 if(j[0] == '#'){
...@@ -50,7 +51,6 @@ const getData = async ()=>{ ...@@ -50,7 +51,6 @@ const getData = async ()=>{
50 let content = await page.content() 51 let content = await page.content()
51 while(true){ 52 while(true){
52 if(temp == content){ 53 if(temp == content){
53 - console.log("finish", result.length)
54 break; 54 break;
55 } 55 }
56 let $ = cheerio.load(content, {decodeEntities: true}) 56 let $ = cheerio.load(content, {decodeEntities: true})
...@@ -68,7 +68,6 @@ const getData = async ()=>{ ...@@ -68,7 +68,6 @@ const getData = async ()=>{
68 temp = content 68 temp = content
69 content = await moveNextPage(page) 69 content = await moveNextPage(page)
70 } 70 }
71 - console.log(result)
72 return result 71 return result
73 } 72 }
74 73
......
...@@ -21,14 +21,16 @@ const makeObject = (array)=>{ ...@@ -21,14 +21,16 @@ const makeObject = (array)=>{
21 result.push({ 21 result.push({
22 title : tempData[0], 22 title : tempData[0],
23 tags : tempData[3].split('#').filter((ele)=>ele != ''), 23 tags : tempData[3].split('#').filter((ele)=>ele != ''),
24 - url : tempData[4] 24 + url : tempData[4],
25 + companyName : "naver"
25 }) 26 })
26 } 27 }
27 else if (tempData.length == 4){ 28 else if (tempData.length == 4){
28 result.push({ 29 result.push({
29 title : tempData[0], 30 title : tempData[0],
30 url : tempData[3], 31 url : tempData[3],
31 - tags : [] 32 + tags : [],
33 + companyName : "naver"
32 }) 34 })
33 } 35 }
34 } 36 }
...@@ -64,7 +66,6 @@ const getData = async ()=>{ ...@@ -64,7 +66,6 @@ const getData = async ()=>{
64 }) 66 })
65 resArr = result.split('</li><li>') 67 resArr = result.split('</li><li>')
66 result = makeObject(resArr) 68 result = makeObject(resArr)
67 - console.log(result)
68 return result; 69 return result;
69 } 70 }
70 71
......
...@@ -20,7 +20,8 @@ const makeObject = (array)=>{ ...@@ -20,7 +20,8 @@ const makeObject = (array)=>{
20 result.push({ 20 result.push({
21 title : tempData[0], 21 title : tempData[0],
22 tags : [], 22 tags : [],
23 - url : tempData[tempData.length - 1] 23 + url : tempData[tempData.length - 1],
24 + companyName : tempData[1]
24 }) 25 })
25 for(let j = 6; j < tempData.length - 1; j++){ 26 for(let j = 6; j < tempData.length - 1; j++){
26 result[result.length -1].tags.push(tempData[j]) 27 result[result.length -1].tags.push(tempData[j])
...@@ -72,11 +73,12 @@ const getData = async ()=>{ ...@@ -72,11 +73,12 @@ const getData = async ()=>{
72 temp = content 73 temp = content
73 content = await moveNextPage(page) 74 content = await moveNextPage(page)
74 } 75 }
75 - console.log(result)
76 return result 76 return result
77 } 77 }
78 78
79 79
80 module.exports = { 80 module.exports = {
81 getData : getData 81 getData : getData
82 -}
...\ No newline at end of file ...\ No newline at end of file
82 +}
83 +
84 +getData()
...\ No newline at end of file ...\ No newline at end of file
......