Toggle navigation
Toggle navigation
This project
Loading...
Sign in
윤준석
/
mamuri-bot
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Snippets
Network
Create a new issue
Builds
Commits
Issue Boards
Authored by
윤준석
2022-05-15 23:25:22 +0900
Browse Files
Options
Browse Files
Download
Email Patches
Plain Diff
Commit
a2f3379e0cc6e1891aacbc4bd2a49fe8c716faa1
a2f3379e
1 parent
d5069fbd
ADD: goroutine in GetItemByKeyword for concurrent execution of crawling
waiting by waitgroup
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
37 additions
and
23 deletions
joongna/service/item.go
joongna/service/item.go
View file @
a2f3379
...
...
@@ -3,6 +3,7 @@ package service
import
(
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"joongna/config"
...
...
@@ -12,6 +13,7 @@ import (
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
...
...
@@ -20,29 +22,27 @@ import (
func
GetItemByKeyword
(
keyword
string
)
([]
model
.
Item
,
error
)
{
var
items
[]
model
.
Item
wg
:=
sync
.
WaitGroup
{}
itemsInfo
:=
getItemsInfoByKeyword
(
keyword
)
for
_
,
itemInfo
:=
range
itemsInfo
{
if
itemInfo
.
CafeName
!=
"중고나라"
{
continue
}
itemUrl
:=
itemInfo
.
Link
sold
,
price
,
thumbnailUrl
,
extraInfo
:=
crawlingNaverCafe
(
itemUrl
)
if
sold
==
"판매 완료"
{
if
itemInfo
.
CafeName
!=
"중고나라"
{
continue
}
item
:=
model
.
Item
{
Platform
:
"중고나라"
,
Name
:
itemInfo
.
Title
,
Price
:
price
,
ThumbnailUrl
:
thumbnailUrl
,
ItemUrl
:
itemUrl
,
ExtraInfo
:
extraInfo
,
wg
.
Add
(
1
)
go
func
(
itemUrl
string
)
{
defer
wg
.
Done
()
err
,
item
:=
crawlingNaverCafe
(
itemUrl
)
if
err
!=
nil
{
log
.
Fatal
(
err
)
}
items
=
append
(
items
,
item
)
items
=
append
(
items
,
*
item
)
}(
itemUrl
)
}
wg
.
Wait
()
return
items
,
nil
}
...
...
@@ -78,32 +78,46 @@ func getItemsInfoByKeyword(keyword string) []model.ApiResponseItem {
return
apiResponse
.
Items
}
func
crawlingNaverCafe
(
cafeUrl
string
)
(
string
,
int
,
string
,
string
)
{
page
:=
rod
.
New
()
.
MustConnect
()
.
MustPage
(
cafeUrl
)
time
.
Sleep
(
time
.
Second
*
1
)
source
:=
page
.
MustElement
(
"iframe#cafe_main"
)
.
MustFrame
()
.
MustHTML
()
func
crawlingNaverCafe
(
cafeUrl
string
)
(
error
,
*
model
.
Item
)
{
frame
:=
rod
.
New
()
.
MustConnect
()
.
MustPage
(
cafeUrl
)
.
MustElement
(
"iframe#cafe_main"
)
time
.
Sleep
(
time
.
Second
*
2
)
source
:=
frame
.
MustFrame
()
.
MustHTML
()
html
,
err
:=
goquery
.
NewDocumentFromReader
(
bytes
.
NewReader
([]
byte
(
source
)))
if
err
!=
nil
{
log
.
Fatal
(
err
)
return
err
,
nil
}
title
:=
html
.
Find
(
"h3.title_text"
)
.
Text
()
sold
:=
html
.
Find
(
"div.sold_area"
)
.
Text
()
price
:=
priceStringToInt
(
html
.
Find
(
".ProductPrice"
)
.
Text
())
thumbnailUrl
,
_
:=
html
.
Find
(
"div.product_thumb img"
)
.
Attr
(
"src"
)
extraInfo
:=
html
.
Find
(
".se-module-text"
)
.
Text
()
title
=
strings
.
TrimSpace
(
title
)
sold
=
strings
.
TrimSpace
(
sold
)
thumbnailUrl
=
strings
.
TrimSpace
(
thumbnailUrl
)
extraInfo
=
strings
.
TrimSpace
(
extraInfo
)
return
sold
,
price
,
thumbnailUrl
,
extraInfo
item
:=
model
.
Item
{
Platform
:
"중고나라"
,
Name
:
title
,
Price
:
price
,
ThumbnailUrl
:
thumbnailUrl
,
ItemUrl
:
cafeUrl
,
ExtraInfo
:
extraInfo
,
}
fmt
.
Println
(
"crawling "
+
cafeUrl
+
" title: "
+
title
)
return
nil
,
&
item
}
func
priceStringToInt
(
priceString
string
)
int
{
strings
.
TrimSpace
(
priceString
)
if
priceString
==
""
{
return
0
}
priceString
=
strings
.
ReplaceAll
(
priceString
,
"원"
,
""
)
priceString
=
strings
.
ReplaceAll
(
priceString
,
","
,
""
)
...
...
Please
register
or
login
to post a comment