다음 웹툰 썸네일이미지 크롤링 성공!

윤혜원
Commit 5e84039dd908e81789b0212f581f6b8a0c02a071 5e84039d 1 parent a5b0a8e9
Showing 2 changed files with 79 additions and 92 deletions
.idea/workspace.xml
routes/index.js
--- a/.idea/workspace.xml
View file @5e84039
+++ b/.idea/workspace.xml
View file @5e84039
@@ -3,10 +3,7 @@
   <component name="ChangeListManager">
     <list default="true" id="ffe1a490-8fdb-4611-8537-4d1c03c40b8d" name="Default" comment="">
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/app.js" beforeDir="false" afterPath="$PROJECT_DIR$/app.js" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/routes/daum.js" beforeDir="false" />
       <change beforePath="$PROJECT_DIR$/routes/index.js" beforeDir="false" afterPath="$PROJECT_DIR$/routes/index.js" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/views/index.ejs" beforeDir="false" afterPath="$PROJECT_DIR$/views/index.ejs" afterDir="false" />
     </list>
     <ignored path="$PROJECT_DIR$/out/" />
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
@@ -58,24 +55,26 @@
     <session id="872999261">
       <usages-collector id="statistics.lifecycle.project">
         <counts>
-          <entry key="project.closed" value="10" />
+          <entry key="project.closed" value="11" />
           <entry key="project.open.time.10" value="1" />
-          <entry key="project.open.time.11" value="2" />
+          <entry key="project.open.time.11" value="3" />
           <entry key="project.open.time.15" value="2" />
           <entry key="project.open.time.16" value="2" />
           <entry key="project.open.time.17" value="1" />
           <entry key="project.open.time.2" value="1" />
           <entry key="project.open.time.6" value="1" />
-          <entry key="project.opened" value="10" />
+          <entry key="project.open.time.8" value="1" />
+          <entry key="project.open.time.9" value="1" />
+          <entry key="project.opened" value="13" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.extensions.open">
         <counts>
           <entry key="css" value="3" />
           <entry key="daum" value="3" />
-          <entry key="ejs" value="32" />
+          <entry key="ejs" value="33" />
           <entry key="iml" value="2" />
-          <entry key="js" value="33" />
+          <entry key="js" value="34" />
           <entry key="json" value="2" />
           <entry key="www" value="1" />
         </counts>
@@ -83,23 +82,23 @@
       <usages-collector id="statistics.file.types.open">
         <counts>
           <entry key="CSS" value="3" />
-          <entry key="EJS" value="35" />
+          <entry key="EJS" value="36" />
           <entry key="IDEA_MODULE" value="2" />
           <entry key="JSON" value="2" />
-          <entry key="JavaScript" value="34" />
+          <entry key="JavaScript" value="35" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.extensions.edit">
         <counts>
           <entry key="daum" value="1" />
           <entry key="ejs" value="117" />
-          <entry key="js" value="2392" />
+          <entry key="js" value="4039" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.types.edit">
         <counts>
           <entry key="EJS" value="118" />
-          <entry key="JavaScript" value="2392" />
+          <entry key="JavaScript" value="4039" />
         </counts>
       </usages-collector>
     </session>
@@ -109,8 +108,8 @@
       <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/routes/mytoons.js">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="190">
+            <state relative-caret-position="1349">
-              <caret line="10" lean-forward="true" selection-start-line="10" selection-end-line="10" />
+              <caret line="71" column="24" selection-start-line="71" selection-start-column="24" selection-end-line="71" selection-end-column="24" />
             </state>
           </provider>
         </entry>
@@ -118,8 +117,8 @@
       <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/routes/setting.js">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="190">
+            <state relative-caret-position="247">
-              <caret line="10" column="21" lean-forward="true" selection-start-line="10" selection-start-column="21" selection-end-line="10" selection-end-column="21" />
+              <caret line="13" column="11" selection-start-line="13" selection-start-column="11" selection-end-line="13" selection-end-column="11" />
             </state>
           </provider>
         </entry>
@@ -127,8 +126,17 @@
       <file pinned="false" current-in-tab="true">
         <entry file="file://$PROJECT_DIR$/routes/index.js">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="171">
+            <state relative-caret-position="19">
-              <caret line="153" column="41" lean-forward="true" selection-start-line="153" selection-start-column="41" selection-end-line="153" selection-end-column="41" />
+              <caret line="188" lean-forward="true" selection-start-line="188" selection-end-line="188" />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/views/index.ejs">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="399">
+              <caret line="21" column="15" lean-forward="true" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
             </state>
           </provider>
         </entry>
@@ -136,8 +144,8 @@
       <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/app.js">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="293">
+            <state relative-caret-position="1425">
-              <caret line="76" column="39" lean-forward="true" selection-start-line="76" selection-start-column="39" selection-end-line="76" selection-end-column="39" />
+              <caret line="75" column="16" lean-forward="true" selection-start-line="75" selection-start-column="16" selection-end-line="75" selection-end-column="16" />
             </state>
           </provider>
         </entry>
@@ -226,8 +234,8 @@
         <option value="$PROJECT_DIR$/views/daum" />
         <option value="$PROJECT_DIR$/views/daum.ejs" />
         <option value="$PROJECT_DIR$/views/index.ejs" />
-        <option value="$PROJECT_DIR$/routes/daum.js" />
         <option value="$PROJECT_DIR$/app.js" />
+        <option value="$PROJECT_DIR$/routes/daum.js" />
         <option value="$PROJECT_DIR$/routes/index.js" />
       </list>
     </option>
@@ -249,9 +257,9 @@
     </packageJsonPaths>
   </component>
   <component name="ProjectFrameBounds" extendedState="6">
-    <option name="x" value="-8" />
+    <option name="x" value="-9" />
-    <option name="width" value="977" />
+    <option name="width" value="978" />
-    <option name="height" value="1038" />
+    <option name="height" value="1039" />
   </component>
   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
   <component name="ProjectView">
@@ -259,7 +267,6 @@
       <foldersAlwaysOnTop value="true" />
     </navigator>
     <panes>
-      <pane id="Scope" />
       <pane id="ProjectPane">
         <subPane>
           <expand>
@@ -297,6 +304,7 @@
           <select />
         </subPane>
       </pane>
+      <pane id="Scope" />
       <pane id="AndroidView" />
       <pane id="PackagesPane" />
     </panes>
@@ -452,6 +460,9 @@
       <workItem from="1543299644956" duration="10319000" />
       <workItem from="1543393354798" duration="25000" />
       <workItem from="1543459526992" duration="19576000" />
+      <workItem from="1543487409434" duration="1283000" />
+      <workItem from="1543489136305" duration="5743000" />
+      <workItem from="1543495038044" duration="387000" />
     </task>
     <task id="LOCAL-00001" summary="초기 commit by intellij">
       <created>1512174269218</created>
@@ -471,7 +482,7 @@
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="211864000" />
+    <option name="totallyTimeSpent" value="219277000" />
   </component>
   <component name="TodoView">
     <todo-panel id="selected-file">
@@ -484,8 +495,9 @@
   </component>
   <component name="ToolWindowManager">
     <frame x="-7" y="-7" width="1550" height="838" extended-state="6" />
+    <editor active="true" />
     <layout>
-      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.48880598" visible="true" weight="0.16523464" />
+      <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.48880598" visible="true" weight="0.16523464" />
       <window_info id="Structure" order="1" weight="0.24964234" />
       <window_info id="npm" order="2" sideWeight="0.51119405" side_tool="true" weight="0.16523606" />
       <window_info id="Image Layers" order="3" />
@@ -679,13 +691,7 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/views/daum">
+    <entry file="file://$PROJECT_DIR$/views/daum" />
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="171">
-          <caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/routes/users.js">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="133">
@@ -693,52 +699,40 @@
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/views/daum.ejs" />
+    <entry file="file://$PROJECT_DIR$/routes/daum.js" />
     <entry file="file://$PROJECT_DIR$/routes/mytoons.js">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="190">
+        <state relative-caret-position="1349">
-          <caret line="10" lean-forward="true" selection-start-line="10" selection-end-line="10" />
+          <caret line="71" column="24" selection-start-line="71" selection-start-column="24" selection-end-line="71" selection-end-column="24" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/views/index.ejs">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="171">
-          <caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/views/daum.ejs">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="350">
-          <caret line="31" column="10" lean-forward="true" selection-start-line="31" selection-start-column="10" selection-end-line="31" selection-end-column="10" />
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/routes/setting.js">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="190">
+        <state relative-caret-position="247">
-          <caret line="10" column="21" lean-forward="true" selection-start-line="10" selection-start-column="21" selection-end-line="10" selection-end-column="21" />
+          <caret line="13" column="11" selection-start-line="13" selection-start-column="11" selection-end-line="13" selection-end-column="11" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/routes/daum.js">
+    <entry file="file://$PROJECT_DIR$/views/index.ejs">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-790">
+        <state relative-caret-position="399">
-          <caret line="107" column="27" lean-forward="true" selection-start-line="107" selection-start-column="27" selection-end-line="107" selection-end-column="27" />
+          <caret line="21" column="15" lean-forward="true" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/app.js">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="293">
+        <state relative-caret-position="1425">
-          <caret line="76" column="39" lean-forward="true" selection-start-line="76" selection-start-column="39" selection-end-line="76" selection-end-column="39" />
+          <caret line="75" column="16" lean-forward="true" selection-start-line="75" selection-start-column="16" selection-end-line="75" selection-end-column="16" />
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/routes/index.js">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="171">
+        <state relative-caret-position="19">
-          <caret line="153" column="41" lean-forward="true" selection-start-line="153" selection-start-column="41" selection-end-line="153" selection-end-column="41" />
+          <caret line="188" lean-forward="true" selection-start-line="188" selection-end-line="188" />
         </state>
       </provider>
     </entry>
--- a/routes/index.js
View file @5e84039
+++ b/routes/index.js
View file @5e84039
@@ -95,48 +95,46 @@ function getAllToons() {
     allWebtoonList = new Array();
 //월요일 다음 웹툰
      var mon='mon';
-    var daum = `http://webtoon.daum.net/data/pc/webtoon/list_serialized/${mon}?timeStamp=1515819276574`;
-    var site='daum';
      var mon_name='MON';
+     var daum = `http://webtoon.daum.net/data/pc/webtoon/list_serialized/${mon}?timeStamp=1515819276574`;
+     var site = 'daum';
         client.fetch(daum, {}, function (err, $, res, body) {
             var data = JSON.parse(body);
             var list = data["data"];
-        console.log(list);
+            list.forEach(function (item, idx) {
-        list.forEach(function(item, idx){
+                var webtoon_link = 'http://webtoon.daum.net/webtoon/view/' + item.nickname.toString();
-            var webtoon_link='http://webtoon.daum.net/webtoon/view/'+item.nickname.toString();
+                var webtoon = {
-            var webtoon= {
                     toon_index: item.id,
-                name : item.title,
+                    name: item.title,
-                thum_link : webtoon_link,
+                    thum_link: item.pcThumbnailImage.url,
-                webtoon_link : webtoon_link,
+                    webtoon_link: webtoon_link,
-                week : mon_name,
+                    week :mon_name,
-                site : site,
+                    site: site,
-                latest : 0
+                    latest: 0
                 };
                 allWebtoonList.push(webtoon);
             });
+
         });
+
 //화요일 다음 웹툰
     var tue='tue';
     var tue_name='TUE';
     var daum1 = `http://webtoon.daum.net/data/pc/webtoon/list_serialized/${tue}?timeStamp=1515819276574`;
-
     client.fetch(daum1, {}, function (err, $, res, body) {
         var data = JSON.parse(body);
         var list = data["data"];
-
         list.forEach(function(item, idx){
             var webtoon_link='http://webtoon.daum.net/webtoon/view/'+item.nickname.toString();
             var webtoon= {
                 toon_index: item.id,
                 name : item.title,
-                thum_link : webtoon_link,
+                thum_link : item.pcThumbnailImage.url,
                 webtoon_link : webtoon_link,
                 week : tue_name,
                 site : site,
                 latest : 0
             };
-
             allWebtoonList.push(webtoon);
         });
@@ -157,7 +155,7 @@ function getAllToons() {
             var webtoon= {
                 toon_index: item.id,
                 name : item.title,
-                thum_link : webtoon_link,
+                thum_link : item.pcThumbnailImage.url,
                 webtoon_link : webtoon_link,
                 week : wed_name,
                 site : site,
@@ -170,7 +168,7 @@ function getAllToons() {
 //목요일 다음 웹툰
     var thu='thu';
-    var daum3 = `http://webtoon.daum.net/data/pc/webtoon/list_serialized/${thu}?timeStamp=1515819276574`;
+    var daum3 =`http://webtoon.daum.net/data/pc/webtoon/list_serialized/${thu}?timeStamp=1515819276574`;
     var thu_name='THU';
     client.fetch(daum3, {}, function (err, $, res, body) {
         var data = JSON.parse(body);
@@ -180,7 +178,7 @@ function getAllToons() {
             var webtoon= {
                 toon_index: item.id,
                 name : item.title,
-                thum_link : webtoon_link,
+                thum_link : item.pcThumbnailImage.url,
                 webtoon_link : webtoon_link,
                 week : thu_name,
                 site : site,
@@ -193,18 +191,17 @@ function getAllToons() {
 //금요일 다음 웹툰
     var fri='fri';
-    var daum4 = `http://webtoon.daum.net/data/pc/webtoon/list_serialized/${fri}?timeStamp=1515819276574`;
+    var daum4 =`http://webtoon.daum.net/data/pc/webtoon/list_serialized/${fri}?timeStamp=1515819276574`;
     var fri_name='FRI';
     client.fetch(daum4, {}, function (err, $, res, body) {
         var data = JSON.parse(body);
-        var list1 = data["data"];
+        var list = data["data"];
-
+        list.forEach(function(item, idx){
-        list1.forEach(function(item, idx){
             var webtoon_link='http://webtoon.daum.net/webtoon/view/'+item.nickname.toString();
             var webtoon= {
                 toon_index: item.id,
                 name : item.title,
-                thum_link : webtoon_link,
+                thum_link : item.pcThumbnailImage.url,
                 webtoon_link : webtoon_link,
                 week : fri_name,
                 site : site,
@@ -217,19 +214,17 @@ function getAllToons() {
 //토요일 다음 웹툰
     var sat='sat';
-    var daum5 = `http://webtoon.daum.net/data/pc/webtoon/list_serialized/${sat}?timeStamp=1515819276574`;
+    var daum5 =`http://webtoon.daum.net/data/pc/webtoon/list_serialized/${sat}?timeStamp=1515819276574`;
     var sat_name='SAT';
     client.fetch(daum5, {}, function (err, $, res, body) {
         var data = JSON.parse(body);
         var list = data["data"];
-
         list.forEach(function(item, idx){
-            //다음 웹툰 아이디, 제목, 요일
             var webtoon_link='http://webtoon.daum.net/webtoon/view/'+item.nickname.toString();
             var webtoon= {
                 toon_index: item.id,
                 name : item.title,
-                thum_link : webtoon_link,
+                thum_link : item.pcThumbnailImage.url,
                 webtoon_link : webtoon_link,
                 week : sat_name,
                 site : site,
@@ -243,18 +238,17 @@ function getAllToons() {
 //일요일 다음 웹툰
     var sun='sun';
     var daum6 = `http://webtoon.daum.net/data/pc/webtoon/list_serialized/${sun}?timeStamp=1515819276574`;
-    var sun_name='SUN'
+    var sun_name='SUN';
     client.fetch(daum6, {}, function (err, $, res, body) {
         var data = JSON.parse(body);
         var list = data["data"];
-
         list.forEach(function(item, idx){
             //다음 웹툰 아이디, 제목, 요일
             var webtoon_link='http://webtoon.daum.net/webtoon/view/'+item.nickname.toString();
             var webtoon= {
                 toon_index: item.id,
                 name : item.title,
-                thum_link : webtoon_link,
+                thum_link : item.pcThumbnailImage.url,
                 webtoon_link : webtoon_link,
                 week : sun_name,
                 site : site,
@@ -262,7 +256,6 @@ function getAllToons() {
             };
             allWebtoonList.push(webtoon);
         });
-
     });
     var allWeeklyToonsUrl = "http://comic.naver.com/webtoon/weekday.nhn";
@@ -276,7 +269,7 @@ function getAllToons() {
                 var thumb_link = $(this).children().first().children().first().attr('src');
                 var name = $(this).next().text();
                 var titleid = webtoon_link.split('?')[1].split('&')[0].split('=')[1];
-                var site = 'naver'
+                var site = 'naver';
                 var webtoon= {
                     toon_index: titleid,
                     name : name,