신일섭 / ShoppingCart

Go to a project
Toggle navigation Toggle navigation pinning
신일섭
45bc2e7a 1 parent b79f14d5

폴더 정리

Inline Side-by-side
Showing 3 changed files with 32 additions and 250 deletions
1 var express = require('express'); 1 var express = require('express');
2 +
3 +// ----------crawling modules---------------
4 +var puppeteer = require('puppeteer');
5 +var gmarket = require('./crawling/gmarket');
6 +var coupang = require('./crawling/coupang');
7 +var auction = require('./crawling/auction');
8 +//------------------------------------------
9 +
2 const request = require('request'); 10 const request = require('request');
3 const TARGET_URL = 'https://api.line.me/v2/bot/message/reply' 11 const TARGET_URL = 'https://api.line.me/v2/bot/message/reply'
4 const TOKEN = 'LFi+FvOOFOydz0EKN22WRnQyiilnEiyxe2HVV1WrI54Jnght6+Gga07m7pxqSQltBJG2AV2lYnYOAk4TqxzZFKW9R36CJcQs+NaNEiNKJtgBfT/t3tEta/J6GtXxGz/DqdHnfZqSSfLFpXzun/ITTwdB04t89/1O/w1cDnyilFU=' 12 const TOKEN = 'LFi+FvOOFOydz0EKN22WRnQyiilnEiyxe2HVV1WrI54Jnght6+Gga07m7pxqSQltBJG2AV2lYnYOAk4TqxzZFKW9R36CJcQs+NaNEiNKJtgBfT/t3tEta/J6GtXxGz/DqdHnfZqSSfLFpXzun/ITTwdB04t89/1O/w1cDnyilFU='
13 +
14 +//-----------------Translating moudules----------------------
5 const PAPAGO_URL = 'https://openapi.naver.com/v1/papago/n2mt' 15 const PAPAGO_URL = 'https://openapi.naver.com/v1/papago/n2mt'
6 const PAPAGO_ID = 'r_kuebFKCqBeL3SI_dFb' 16 const PAPAGO_ID = 'r_kuebFKCqBeL3SI_dFb'
7 const PAPAGO_SECRET = 'Wdz2tudrzB' 17 const PAPAGO_SECRET = 'Wdz2tudrzB'
18 +var trans = require('./trans/trans')
19 +//-----------------------------------------------------------
20 +
8 const fs = require('fs'); 21 const fs = require('fs');
9 const path = require('path'); 22 const path = require('path');
10 const HTTPS = require('https'); 23 const HTTPS = require('https');
...@@ -12,86 +25,46 @@ const domain = "www.chatbotshin.tk" ...@@ -12,86 +25,46 @@ const domain = "www.chatbotshin.tk"
12 const sslport = 23023; 25 const sslport = 23023;
13 const bodyParser = require('body-parser'); 26 const bodyParser = require('body-parser');
14 27
15 -// target language verification 28 +//-----target language verification--------------
16 -var trans_dest = undefined; 29 +var trans_dest = 'target=en'; // defualt = english
30 +var trans_trigger = 0;
31 +//-----------------------------------------------
17 32
18 var app = express(); 33 var app = express();
34 +
35 +//------middlewares------------------------------
19 app.use(bodyParser.json()); 36 app.use(bodyParser.json());
37 +
38 +//--------------------Utility Selector----------------------------------------------------------------
20 app.post('/hook', function (req, res) { 39 app.post('/hook', function (req, res) {
21 40
41 + //-------- request values ---------
22 var eventObj = req.body.events[0]; 42 var eventObj = req.body.events[0];
23 var source = eventObj.source; 43 var source = eventObj.source;
24 var message = eventObj.message; 44 var message = eventObj.message;
45 + //---------------------------------
25 46
26 - // request log 47 + //---------------- request log------------------------------------------------
27 console.log('======================', new Date() ,'======================'); 48 console.log('======================', new Date() ,'======================');
28 console.log('[request]', req.body); 49 console.log('[request]', req.body);
29 console.log('[request source] ', eventObj.source); 50 console.log('[request source] ', eventObj.source);
30 console.log('[request message]', eventObj.message); 51 console.log('[request message]', eventObj.message);
52 + //----------------------------------------------------------------------------
31 53
32 // language identification & translation 54 // language identification & translation
33 - if(eventObj.message.text=='영어' || eventObj.message.text=='프랑스어' || eventObj.message.text=='일본어'){ 55 + if(eventObj.message.text.indexOf('/번역')){
34 - trans_change(eventObj.message.text); 56 + trans.trans_lng_selector(); // trans_trigger -> 1
35 - console.log('[trans_dest]', trans_dest);
36 } 57 }
37 - else if(trans_dest != undefined){ 58 + else if(trans_trigger==1){
38 - trans(eventObj.replyToken, eventObj.message.text); 59 + translating(eventObj.replyToken, eventObj.message.text);
39 } 60 }
40 61
41 - res.sendStatus(200);
42 -});
43 62
44 -function trans(replyToken, message) {
45 -
46 - request.post(
47 - {
48 - url: PAPAGO_URL,
49 - headers: {
50 - 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
51 - 'X-Naver-Client-Id': `${PAPAGO_ID}`,
52 - 'X-Naver-Client-Secret': `${PAPAGO_SECRET}`
53 - },
54 - body: 'source=ko&' + trans_dest + '&text=' + message,
55 - json:true
56 - },(error, response, body) => {
57 - if(!error && response.statusCode == 200) {
58 - console.log(body.message);
59 - var transMessage = body.message.result.translatedText;
60 - request.post(
61 - {
62 - url: TARGET_URL,
63 - headers: {
64 - 'Authorization': `Bearer ${TOKEN}`
65 - },
66 - json: {
67 - "replyToken":replyToken,
68 - "messages":[
69 - {
70 - "type":"text",
71 - "text":transMessage
72 - }
73 - ]
74 - }
75 - },(error, response, body) => {
76 - console.log(body)
77 - });
78 - }
79 - });
80 63
81 -}
82 64
83 -// language index identification 65 + res.sendStatus(200);
84 -function trans_change(index){ 66 +});
85 - if(index == '영어'){ 67 +
86 - trans_dest = 'target=en';
87 - }
88 - else if(index == '일본어'){
89 - trans_dest = 'target=ja';
90 - }
91 - else if(index == '프랑스어'){
92 - trans_dest = 'target=fr';
93 - }
94 -}
95 68
96 try { 69 try {
97 const option = { 70 const option = {
......
1 -const puppeteer = require('puppeteer');
2 -
3 -//auction_c();
4 -
5 -// auction Cart Crawling
6 -async function auction_c(){
7 -
8 - // launching headless browser
9 - const browser = await puppeteer.launch();
10 - // making a new page
11 - const page = await browser.newPage();
12 -
13 - //console input id & password
14 - var args = process.argv;
15 -
16 - var a_id = args[2];
17 - var a_pw = args[3];
18 -
19 - console.log(args[2] + " " + args[3]);
20 -
21 - // Gmarket login page
22 - await page.goto('https://memberssl.auction.co.kr/Authenticate/?url=http%3a%2f%2fwww.auction.co.kr%2f&return_value=0');
23 - // Ading user information
24 - await page.evaluate((id, pwd) => {
25 - document.querySelector('#id').value = id;
26 - document.querySelector('#password').value = pwd;
27 - }, a_id, a_pw);
28 -
29 - await console.log(a_id,a_pw);
30 -
31 - // try login
32 - await page.click('.btn_login');
33 - await page.waitForNavigation();
34 -
35 - // goto cart page
36 - await page.goto('https://cart.auction.co.kr/ko/cart');
37 - //await page.screenshot({ path: 'auction.png', fullPage:true });
38 -
39 - // container which will hold crawled data [{},{}...]
40 - let data = [];
41 -
42 - // crawling start! (using getOne and getAll function)
43 - data = await getAll(page);
44 - //data.push(await getAll(page)); // data[0][n]
45 -
46 - //logging the result
47 - for(let index = 0; index < data.length; index++){
48 - console.log(data[index]);
49 - }
50 -
51 - await browser.close();
52 -
53 -}
54 -
55 -
56 -
57 - // Crawling cart informations(object) to data(array)
58 - async function getAll(page) {
59 - var data = [];
60 -
61 - //const number = await page.$$eval("#cart_list > ol > li:nth-child(1) > div.cart--basket_body > div > ul > li", (data) => data.length);
62 - const number = await page.$$eval("#cart__body > div > div.box__cart-list > ul > li", (data) => data.length);
63 - // counting the number of the box
64 - for (let index = 0; index < number; index++) {
65 - data.push(await getOne(page, index + 1));
66 - // pushing to the array
67 -
68 - }
69 -
70 - return Promise.resolve(data);
71 -}
72 -
73 -
74 -// Crawling cart information to data(object)
75 -async function getOne(page, index) {
76 -
77 - var data = {};
78 -
79 - // this is example code
80 - //data.programPeriod = await page.$eval("#iph_content > div > div.list_type_h1.web_view.mt3 > table > tbody > tr:nth-child(" + index + ") > td:nth-child(5)", (data) => data.textContent);
81 -
82 - // product name
83 - data.prd_name = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-info > dl > dd > div.box__title > a > span", data => data.textContent);
84 - // product price
85 - data.prd_price = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-info > dl > dd > div.box__unit-price > div > span > strong", data => data.textContent);
86 - // product link
87 - data.prd_link = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-info > dl > dd > div.box__title > a", data => data.href);
88 - // product image source
89 - data.prd_img = await page.$eval("#cart__body > div > div.box__cart-list > ul > li:nth-child(" + index + ") > div > div.box__group-seller-body > div > div.box__group-item > ul > li > div > div.box__item-img > a > img", data => data.src);
90 - // not yet
91 -
92 - return Promise.resolve(data);
93 -}
...\ No newline at end of file ...\ No newline at end of file
1 -const puppeteer = require('puppeteer');
2 -const readline = require('readline');
3 -
4 -//gmarket_c();
5 -
6 -// Gmarket Cart Crawling
7 -async function gmarket_c(){
8 -
9 - // launching headless browser
10 - const browser = await puppeteer.launch();
11 - // making a new page
12 - const page = await browser.newPage();
13 -
14 - //console input id & password
15 - var args = process.argv;
16 -
17 - var g_id = args[2];
18 - var g_pw = args[3];
19 -
20 - console.log(args[2] + " " + args[3]);
21 -
22 - // Gmarket login page
23 - await page.goto('https://signinssl.gmarket.co.kr/login/login?url=https://www.gmarket.co.kr/');
24 - // Ading user information
25 - await page.evaluate((id, pwd) => {
26 - document.querySelector('#id').value = id;
27 - document.querySelector('#pwd').value = pwd;
28 - }, g_id, g_pw);
29 -
30 - await console.log(g_id,g_pw);
31 -
32 - // try login
33 - await page.click('.button_login');
34 - await page.waitForNavigation();
35 -
36 - // goto cart page
37 - await page.goto('https://cart.gmarket.co.kr/ko/cart');
38 - //await page.screenshot({ path: 'gmarket.png', fullPage:true });
39 -
40 - // container which will hold crawled data [{},{}...]
41 - let data = [];
42 -
43 - // crawling start! (using getOne and getAll function)
44 - data = await getAll(page);
45 - //data.push(await getAll(page)); // data[0][n]
46 -
47 - //logging the result
48 - for(let index = 0; index < data.length; index++){
49 - console.log(data[index]);
50 - }
51 -
52 - await browser.close();
53 -
54 -}
55 -
56 -
57 -
58 - // Crawling cart informations(object) to data(array)
59 - async function getAll(page) {
60 - var data = [];
61 -
62 - //const number = await page.$$eval("#cart_list > ol > li:nth-child(1) > div.cart--basket_body > div > ul > li", (data) => data.length);
63 - const number = await page.$$eval("#cart_list > ol > li", (data) => data.length);
64 - // counting the number of the box
65 - for (let index = 0; index < number; index++) {
66 - data.push(await getOne(page, index + 1));
67 - // pushing to the array
68 -
69 - }
70 -
71 - return Promise.resolve(data);
72 -}
73 -
74 -
75 -// Crawling cart information to data(object)
76 -async function getOne(page, index) {
77 -
78 - var data = {};
79 -
80 - // this is example code
81 - //data.programPeriod = await page.$eval("#iph_content > div > div.list_type_h1.web_view.mt3 > table > tbody > tr:nth-child(" + index + ") > td:nth-child(5)", (data) => data.textContent);
82 -
83 - // product name
84 - data.prd_name = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_title > a > span", data => data.textContent);
85 - // product price
86 - data.prd_price = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_price > span.format-price > span > strong", data => data.textContent);
87 - // product link
88 - data.prd_link = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_title > a", data => data.href);
89 - // product image source
90 - data.prd_img = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_img > a > img", data => data.src);
91 - // not yet
92 -
93 - return Promise.resolve(data);
94 -}
95 -
96 -module.exports.gmarket_c = gmarket_c;
97 -module.exports.getOne = getOne;
98 -module.exports.getAll = getAll;
...\ No newline at end of file ...\ No newline at end of file