신일섭 / ShoppingCart

Go to a project
Toggle navigation Toggle navigation pinning
신일섭
45bc2e7a 1 parent b79f14d5

폴더 정리

Inline Side-by-side
Showing 3 changed files with 32 additions and 250 deletions
var express = require('express');
// ----------crawling modules---------------
var puppeteer = require('puppeteer');
var gmarket = require('./crawling/gmarket');
var coupang = require('./crawling/coupang');
var auction = require('./crawling/auction');
//------------------------------------------
const request = require('request');
const TARGET_URL = 'https://api.line.me/v2/bot/message/reply'
const TOKEN = 'LFi+FvOOFOydz0EKN22WRnQyiilnEiyxe2HVV1WrI54Jnght6+Gga07m7pxqSQltBJG2AV2lYnYOAk4TqxzZFKW9R36CJcQs+NaNEiNKJtgBfT/t3tEta/J6GtXxGz/DqdHnfZqSSfLFpXzun/ITTwdB04t89/1O/w1cDnyilFU='
//-----------------Translating moudules----------------------
const PAPAGO_URL = 'https://openapi.naver.com/v1/papago/n2mt'
const PAPAGO_ID = 'r_kuebFKCqBeL3SI_dFb'
const PAPAGO_SECRET = 'Wdz2tudrzB'
var trans = require('./trans/trans')
//-----------------------------------------------------------
const fs = require('fs');
const path = require('path');
const HTTPS = require('https');
......@@ -12,86 +25,46 @@ const domain = "www.chatbotshin.tk"
const sslport = 23023;
const bodyParser = require('body-parser');
// target language verification
var trans_dest = undefined;
//-----target language verification--------------
var trans_dest = 'target=en'; // defualt = english
var trans_trigger = 0;
//-----------------------------------------------
var app = express();
//------middlewares------------------------------
app.use(bodyParser.json());
//--------------------Utility Selector----------------------------------------------------------------
app.post('/hook', function (req, res) {
//-------- request values ---------
var eventObj = req.body.events[0];
var source = eventObj.source;
var message = eventObj.message;
//---------------------------------
// request log
//---------------- request log------------------------------------------------
console.log('======================', new Date() ,'======================');
console.log('[request]', req.body);
console.log('[request source] ', eventObj.source);
console.log('[request message]', eventObj.message);
//----------------------------------------------------------------------------
// language identification & translation
if(eventObj.message.text=='영어' || eventObj.message.text=='프랑스어' || eventObj.message.text=='일본어'){
trans_change(eventObj.message.text);
console.log('[trans_dest]', trans_dest);
if(eventObj.message.text.indexOf('/번역')){
trans.trans_lng_selector(); // trans_trigger -> 1
}
else if(trans_dest != undefined){
trans(eventObj.replyToken, eventObj.message.text);
else if(trans_trigger==1){
translating(eventObj.replyToken, eventObj.message.text);
}
res.sendStatus(200);
});
function trans(replyToken, message) {
request.post(
{
url: PAPAGO_URL,
headers: {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'X-Naver-Client-Id': `${PAPAGO_ID}`,
'X-Naver-Client-Secret': `${PAPAGO_SECRET}`
},
body: 'source=ko&' + trans_dest + '&text=' + message,
json:true
},(error, response, body) => {
if(!error && response.statusCode == 200) {
console.log(body.message);
var transMessage = body.message.result.translatedText;
request.post(
{
url: TARGET_URL,
headers: {
'Authorization': `Bearer ${TOKEN}`
},
json: {
"replyToken":replyToken,
"messages":[
{
"type":"text",
"text":transMessage
}
]
}
},(error, response, body) => {
console.log(body)
});
}
});
}
// language index identification
function trans_change(index){
if(index == '영어'){
trans_dest = 'target=en';
}
else if(index == '일본어'){
trans_dest = 'target=ja';
}
else if(index == '프랑스어'){
trans_dest = 'target=fr';
}
}
res.sendStatus(200);
});
try {
const option = {
......
const puppeteer = require('puppeteer');
//auction_c();
// auction Cart Crawling
async function auction_c(){
// launching headless browser
const browser = await puppeteer.launch();
// making a new page
const page = await browser.newPage();
//console input id & password
var args = process.argv;
var a_id = args[2];
var a_pw = args[3];
console.log(args[2] + " " + args[3]);
// Gmarket login page
await page.goto('https://memberssl.auction.co.kr/Authenticate/?url=http%3a%2f%2fwww.auction.co.kr%2f&return_value=0');
// Ading user information
await page.evaluate((id, pwd) => {
document.querySelector('#id').value = id;
document.querySelector('#password').value = pwd;
}, a_id, a_pw);
await console.log(a_id,a_pw);
// try login
await page.click('.btn_login');
await page.waitForNavigation();
// goto cart page
await page.goto('https://cart.auction.co.kr/ko/cart');
//await page.screenshot({ path: 'auction.png', fullPage:true });
// container which will hold crawled data [{},{}...]
let data = [];
// crawling start! (using getOne and getAll function)
data = await getAll(page);
//data.push(await getAll(page)); // data[0][n]
//logging the result
for(let index = 0; index < data.length; index++){
console.log(data[index]);
}
await browser.close();
}
// Crawling cart informations(object) to data(array)
async function getAll(page) {
var data = [];
//const number = await page.$$eval("#cart_list > ol > li:nth-child(1) > div.cart--basket_body > div > ul > li", (data) => data.length);
const number = await page.$$eval("#cart__body > div > div.box__cart-list > ul > li", (data) => data.length);
// counting the number of the box
for (let index = 0; index < number; index++) {
data.push(await getOne(page, index + 1));
// pushing to the array
}
return Promise.resolve(data);
}
// Crawling cart information to data(object)
async function getOne(page, index) {
var data = {};
// this is example code
//data.programPeriod = await page.$eval("#iph_content > div > div.list_type_h1.web_view.mt3 > table > tbody > tr:nth-child(" + index + ") > td:nth-child(5)", (data) => data.textContent);
// product name
data.prd_name = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-info > dl > dd > div.box__title > a > span", data => data.textContent);
// product price
data.prd_price = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-info > dl > dd > div.box__unit-price > div > span > strong", data => data.textContent);
// product link
data.prd_link = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-info > dl > dd > div.box__title > a", data => data.href);
// product image source
data.prd_img = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-img > a > img", data => data.src);
// not yet
return Promise.resolve(data);
}
\ No newline at end of file
const puppeteer = require('puppeteer');
const readline = require('readline');
//gmarket_c();
// Gmarket Cart Crawling
async function gmarket_c(){
// launching headless browser
const browser = await puppeteer.launch();
// making a new page
const page = await browser.newPage();
//console input id & password
var args = process.argv;
var g_id = args[2];
var g_pw = args[3];
console.log(args[2] + " " + args[3]);
// Gmarket login page
await page.goto('https://signinssl.gmarket.co.kr/login/login?url=https://www.gmarket.co.kr/');
// Ading user information
await page.evaluate((id, pwd) => {
document.querySelector('#id').value = id;
document.querySelector('#pwd').value = pwd;
}, g_id, g_pw);
await console.log(g_id,g_pw);
// try login
await page.click('.button_login');
await page.waitForNavigation();
// goto cart page
await page.goto('https://cart.gmarket.co.kr/ko/cart');
//await page.screenshot({ path: 'gmarket.png', fullPage:true });
// container which will hold crawled data [{},{}...]
let data = [];
// crawling start! (using getOne and getAll function)
data = await getAll(page);
//data.push(await getAll(page)); // data[0][n]
//logging the result
for(let index = 0; index < data.length; index++){
console.log(data[index]);
}
await browser.close();
}
// Crawling cart informations(object) to data(array)
async function getAll(page) {
var data = [];
//const number = await page.$$eval("#cart_list > ol > li:nth-child(1) > div.cart--basket_body > div > ul > li", (data) => data.length);
const number = await page.$$eval("#cart_list > ol > li", (data) => data.length);
// counting the number of the box
for (let index = 0; index < number; index++) {
data.push(await getOne(page, index + 1));
// pushing to the array
}
return Promise.resolve(data);
}
// Crawling cart information to data(object)
async function getOne(page, index) {
var data = {};
// this is example code
//data.programPeriod = await page.$eval("#iph_content > div > div.list_type_h1.web_view.mt3 > table > tbody > tr:nth-child(" + index + ") > td:nth-child(5)", (data) => data.textContent);
// product name
data.prd_name = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_title > a > span", data => data.textContent);
// product price
data.prd_price = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_price > span.format-price > span > strong", data => data.textContent);
// product link
data.prd_link = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_title > a", data => data.href);
// product image source
data.prd_img = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_img > a > img", data => data.src);
// not yet
return Promise.resolve(data);
}
module.exports.gmarket_c = gmarket_c;
module.exports.getOne = getOne;
module.exports.getAll = getAll;
\ No newline at end of file