김대선

파일 저장이 안됐을 때, read() 호출하면 다시 save() 구현, save가 너무 많이 호출되는 것을 막기위한 status 변수 사용

const fs = require('fs')
const functions = require('./function')
let status = false
const save = async()=> {
if(status){
return
}
else if(!status){
status = true
}
try {
let string = ""
let data = await functions.getKakaoData()
let temp = ""
for(let i of data){
temp = "title : " + i.title
string = string + temp + "\n"
temp = "tags : " + i.tags.toString()
string = string + temp + "\n"
temp = "url : " + i.url
string = string + temp + "\n"
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
data = await functions.getNaverFunction()
temp = ""
for(let i of data){
temp = "title : " + i.title
string = string + temp + "\n"
temp = "tags : " + i.tags.toString()
string = string + temp + "\n"
temp = "url : " + i.url
string = string + temp + "\n"
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
data = await functions.getProgrammersFunction()
temp = ""
for(let i of data){
temp = "title : " + i.title
string = string + temp + "\n"
temp = "tags : " + i.tags.toString()
string = string + temp + "\n"
temp = "url : " + i.url
string = string + temp + "\n"
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
const today = new Date()
fs.writeFile(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, string, 'utf-8', (err)=>{
status = false
if(err){
console.log("파일저장시에 오류")
}
else console.log("저장완료!")
})
} catch (error) {
status = false
console.log("파일저장시에 오류")
}
}
const read = () =>{
const data = []
let title = ""
......@@ -13,6 +74,7 @@ const read = () =>{
try {
string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, 'utf-8', 'r')
} catch (error) {
save()
if(today.getDate()-1 == 0){
if([1, 3, 5, 7, 8, 10, 12].includes(today.getMonth() - 1)){
string = fs.readFileSync(`./datas/${today.getFullYear()}.${today.getMonth() - 1}.${31}`, 'utf-8', 'r')
......@@ -57,63 +119,6 @@ const read = () =>{
return data
}
const save = async()=> {
try {
let string = ""
let data = await functions.getKakaoData()
let temp = ""
for(let i of data){
temp = "title : " + i.title
string = string + temp + "\n"
temp = "tags : " + i.tags.toString()
string = string + temp + "\n"
temp = "url : " + i.url
string = string + temp + "\n"
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
data = await functions.getNaverFunction()
temp = ""
for(let i of data){
temp = "title : " + i.title
string = string + temp + "\n"
temp = "tags : " + i.tags.toString()
string = string + temp + "\n"
temp = "url : " + i.url
string = string + temp + "\n"
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
data = await functions.getProgrammersFunction()
temp = ""
for(let i of data){
temp = "title : " + i.title
string = string + temp + "\n"
temp = "tags : " + i.tags.toString()
string = string + temp + "\n"
temp = "url : " + i.url
string = string + temp + "\n"
temp = "companyName : " + i.companyName
string = string + temp + "\n"
}
const today = new Date()
fs.writeFile(`./datas/${today.getFullYear()}.${today.getMonth()}.${today.getDate()}`, string, 'utf-8', (err)=>{
if(err){
console.log("파일저장시에 오류")
console.log(err)
save()
}
else console.log("저장완료!")
})
} catch (error) {
console.log("데이터 가져오는 과정에서 오류")
console.log(error)
save()
}
}
module.exports = {
save : save,
read : read
......
......@@ -57,13 +57,10 @@ const getData = async ()=>{
decodeEntities : true
}
}).replace(/(<([^>]+)>)*(\\t)?/gi, "")
console.log(final)
while(true){
await page.goto(`https://programmers.co.kr/job?page=${count}`)
content = await page.content()
console.log( final, count)
if(final < count){
console.log("finish", result.length)
break;
}
$ = cheerio.load(content, {decodeEntities: true})
......@@ -78,7 +75,6 @@ const getData = async ()=>{
}
})
if(item ==''){
console.log("break!!!!")
break;
}
item = item.split("</div>`")
......@@ -88,7 +84,7 @@ const getData = async ()=>{
result = result.concat(await makeObject(resArr))
count = count + 1
}
console.log(result)
console.log("kakao : ", result.length)
return result
}
......