신일섭

G마켓, 쿠팡 크롤링 기능 업데이트

const puppeteer = require('puppeteer');
// Gmarket Cart Crawling
(async () => {
// launching headless browser
const browser = await puppeteer.launch();
// making a new page
const page = await browser.newPage();
//console input id & password
var args = process.argv;
var c_id = args[2];
var c_pw = args[3];
console.log(args[2] + " " + args[3]);
// Gmarket login page
await page.goto('https://login.coupang.com/login/login.pang?rtnUrl=https%3A%2F%2Fwww.coupang.com%2Fnp%2Fpost%2Flogin%3Fr%3Dhttps%253A%252F%252Fwww.coupang.com%252F');
// Ading user information
await page.evaluate((id, pwd) => {
document.querySelector('#login-email-input').value = id;
document.querySelector('#login-password-input').value = pwd;
}, c_id, c_pw);
await console.log(c_id,c_pw);
// try login
await page.click('.login__button');
await page.waitForNavigation();
// goto cart page
await page.goto('https://cart.coupang.com/cartView.pang');
await page.screenshot({ path: 'coupang.png', fullPage:true });
// container which will hold crawled data [{},{}...]
let data = [];
// crawling start! (using getOne and getAll function)
data = await getAll(page);
//data.push(await getAll(page)); // data[0][n]
//logging the result
for(let index = 0; index < data.length; index++){
console.log(data[index]);
}
await browser.close();
})();
// Crawling cart informations(object) to data(array)
async function getAll(page) {
var data = [];
//coupang table tr counts has dummy 3 tr
const number = await page.$$eval("#cartTable-sku > tr", (data) => data.length)-3;
// counting the number of the box
// coupang car info starts with index 2
for (let index = 0; index < number; index++) {
data.push(await getOne(page, index + 2));
// pushing to the array
}
return Promise.resolve(data);
}
// Crawling cart information to data(object)
async function getOne(page, index) {
var data = {};
// this is example code
//data.programPeriod = await page.$eval("#iph_content > div > div.list_type_h1.web_view.mt3 > table > tbody > tr:nth-child(" + index + ") > td:nth-child(5)", (data) => data.textContent);
// product name
data.prd_name = await page.$eval("#cartTable-sku > tr:nth-child(" + index + ") > td.product-box > div.product-name-part > a", data => data.textContent);
// product price
data.prd_price = await page.$eval("#cartTable-sku > tr:nth-child(" + index + ") > td.unit-total-price > div", data => data.textContent);
// product link
data.prd_link = await page.$eval("#cartTable-sku > tr:nth-child(" + index + ") > td.product-box > div.product-name-part > a", data => data.href);
// product image source
data.prd_img = await page.$eval("#cartTable-sku > tr:nth-child(" + index + ") > td:nth-child(2) > a > img", data => data.src);
// not yet
return Promise.resolve(data);
}
\ No newline at end of file
const puppeteer = require('puppeteer');
const readline = require('readline');
// Gmarket Cart Crawling
(async () => {
// launching headless browser
const browser = await puppeteer.launch();
// making a new page
const page = await browser.newPage();
//console input id & password
var args = process.argv;
var g_id = args[2];
var g_pw = args[3];
console.log(args[2] + " " + args[3]);
// Gmarket login page
await page.goto('https://signinssl.gmarket.co.kr/login/login?url=https://www.gmarket.co.kr/');
// Ading user information
await page.evaluate((id, pwd) => {
document.querySelector('#id').value = id;
document.querySelector('#pwd').value = pwd;
}, g_id, g_pw);
await console.log(g_id,g_pw);
// try login
await page.click('.button_login');
await page.waitForNavigation();
// goto cart page
await page.goto('https://cart.gmarket.co.kr/ko/cart');
await page.screenshot({ path: 'gmarket.png', fullPage:true });
// container which will hold crawled data [{},{}...]
let data = [];
// crawling start! (using getOne and getAll function)
data = await getAll(page);
//data.push(await getAll(page)); // data[0][n]
//logging the result
for(let index = 0; index < data.length; index++){
console.log(data[index]);
}
await browser.close();
})();
// Crawling cart informations(object) to data(array)
async function getAll(page) {
var data = [];
//const number = await page.$$eval("#cart_list > ol > li:nth-child(1) > div.cart--basket_body > div > ul > li", (data) => data.length);
const number = await page.$$eval("#cart_list > ol > li", (data) => data.length);
// counting the number of the box
for (let index = 0; index < number; index++) {
data.push(await getOne(page, index + 1));
// pushing to the array
}
return Promise.resolve(data);
}
// Crawling cart information to data(object)
async function getOne(page, index) {
var data = {};
// this is example code
//data.programPeriod = await page.$eval("#iph_content > div > div.list_type_h1.web_view.mt3 > table > tbody > tr:nth-child(" + index + ") > td:nth-child(5)", (data) => data.textContent);
// product name
data.prd_name = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_title > a > span", data => data.textContent);
// product price
data.prd_price = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_price > span.format-price > span > strong", data => data.textContent);
// product link
data.prd_link = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_info > dl > dd > div.section.item_title > a", data => data.href);
// product image source
data.prd_img = await page.$eval("#cart_list > ol > li:nth-child(" + index + ") > div.cart--basket_body > div > ul > li > div > div.item_img > a > img", data => data.src);
// not yet
return Promise.resolve(data);
}
\ No newline at end of file
{
"name": "shoppingcart",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@types/node": {
"version": "14.0.9",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.9.tgz",
"integrity": "sha512-0sCTiXKXELOBxvZLN4krQ0FPOAA7ij+6WwvD0k/PHd9/KAkr4dXel5J9fh6F4x1FwAQILqAWkmpeuS6mjf1iKA==",
"optional": true
},
"@types/yauzl": {
"version": "2.9.1",
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz",
"integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==",
"optional": true,
"requires": {
"@types/node": "*"
}
},
"agent-base": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz",
"integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g=="
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"base64-js": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
},
"bl": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
"integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
"requires": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
}
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"buffer": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
"integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4"
}
},
"buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
},
"chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": {
"ms": "^2.1.1"
}
},
"end-of-stream": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
"requires": {
"once": "^1.4.0"
}
},
"extract-zip": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.0.tgz",
"integrity": "sha512-i42GQ498yibjdvIhivUsRslx608whtGoFIhF26Z7O4MYncBxp8CwalOs1lnHy21A9sIohWO2+uiE4SRtC9JXDg==",
"requires": {
"@types/yauzl": "^2.9.1",
"debug": "^4.1.1",
"get-stream": "^5.1.0",
"yauzl": "^2.10.0"
}
},
"fd-slicer": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
"integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
"requires": {
"pend": "~1.2.0"
}
},
"fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"get-stream": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
"integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
"requires": {
"pump": "^3.0.0"
}
},
"glob": {
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"https-proxy-agent": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
"integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==",
"requires": {
"agent-base": "5",
"debug": "4"
}
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"requires": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"mime": {
"version": "2.4.6",
"resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz",
"integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA=="
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
},
"pend": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
"integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
},
"progress": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
},
"proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"pump": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
},
"puppeteer": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-3.3.0.tgz",
"integrity": "sha512-23zNqRltZ1PPoK28uRefWJ/zKb5Jhnzbbwbpcna2o5+QMn17F0khq5s1bdH3vPlyj+J36pubccR8wiNA/VE0Vw==",
"requires": {
"debug": "^4.1.0",
"extract-zip": "^2.0.0",
"https-proxy-agent": "^4.0.0",
"mime": "^2.0.3",
"progress": "^2.0.1",
"proxy-from-env": "^1.0.0",
"rimraf": "^3.0.2",
"tar-fs": "^2.0.0",
"unbzip2-stream": "^1.3.3",
"ws": "^7.2.3"
}
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
},
"readline": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz",
"integrity": "sha1-xYDXfvLPyHUrEySYBg3JeTp6wBw="
},
"rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
"requires": {
"glob": "^7.1.3"
}
},
"safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
},
"string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"requires": {
"safe-buffer": "~5.2.0"
}
},
"tar-fs": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz",
"integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==",
"requires": {
"chownr": "^1.1.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.0.0"
}
},
"tar-stream": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz",
"integrity": "sha512-UaF6FoJ32WqALZGOIAApXx+OdxhekNMChu6axLJR85zMMjXKWFGjbIRe+J6P4UnRGg9rAwWvbTT0oI7hD/Un7Q==",
"requires": {
"bl": "^4.0.1",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
}
},
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"unbzip2-stream": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
"integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
"requires": {
"buffer": "^5.2.1",
"through": "^2.3.8"
}
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"ws": {
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.3.0.tgz",
"integrity": "sha512-iFtXzngZVXPGgpTlP1rBqsUK82p9tKqsWRPg5L56egiljujJT3vGAYnHANvFxBieXrTFavhzhxW52jnaWV+w2w=="
},
"yauzl": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
"integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
"requires": {
"buffer-crc32": "~0.2.3",
"fd-slicer": "~1.1.0"
}
}
}
}
{
"name": "shoppingcart",
"version": "1.0.0",
"description": "기록날짜 오후 1:30 2020-05-13",
"main": "coupang.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "ssh://git@khuhub.khu.ac.kr:12959/2015104185/ShoppingCart.git"
},
"author": "",
"license": "ISC",
"dependencies": {
"puppeteer": "^3.3.0",
"readline": "^1.3.0"
}
}