Toggle navigation
Toggle navigation
This project
Loading...
Sign in
유명현
/
mamuri-bot
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Snippets
Network
Create a new issue
Builds
Commits
Issue Boards
Authored by
윤준석
2022-05-23 00:54:23 +0900
Browse Files
Options
Browse Files
Download
Email Patches
Plain Diff
Commit
65a7273bbedb437547c2f795ec219167629101f2
65a7273b
1 parent
d470fc86
FIX: joongna crawling -> naver cafe api
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
48 additions
and
66 deletions
joongna/config/config.go
joongna/model/api_response.go
joongna/service/item.go
joongna/config/config.go
View file @
65a7273
...
...
@@ -12,6 +12,11 @@ type Config struct {
CLIENTID
string
`env:"SECRET.CLIENTID"`
CLIENTSECRET
string
`env:"SECRET.CLIENTSECRET"`
}
Header
struct
{
Cookie
string
`env:"HEADER.COOKIE"`
UserAgent
string
`env:"HEADER.USERAGENT"`
}
}
var
Cfg
*
Config
...
...
joongna/model/api_response.go
View file @
65a7273
package
model
type
ApiResponse
struct
{
LastBuildDate
string
`json:"lastBuildDate"`
Total
uint
`json:"total"`
Start
uint
`json:"start"`
Display
uint
`json:"display"`
Items
[]
ApiResponseItem
`json:"items"`
CafeId
int
`json:"cafeId"`
ArticelCount
int
`json:"articleCount"`
Query
string
`json:"query"`
Items
[]
ApiResponseItem
`json:"articleList"`
}
type
ApiResponseItem
struct
{
Title
string
`json:"title"`
Link
string
`json:"link"`
Description
string
`json:"description"`
CafeName
string
`json:"cafename"`
ArticleId
int
`json:"articleId"`
Title
string
`json:"subject"`
ExtraInfo
string
`json:"summary"`
ThumbnailUrl
string
`json:"thumbnailImageUrl"`
ProductSale
ApiResponseItemSale
`json:"productSale"`
}
type
ApiResponseItemSale
struct
{
SaleStatus
string
`json:"saleStatue"`
Cost
string
`json:"cost"`
}
...
...
joongna/service/item.go
View file @
65a7273
package
service
import
(
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"joongna/config"
...
...
@@ -13,36 +13,35 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
)
func
GetItemByKeyword
(
keyword
string
)
([]
model
.
Item
,
error
)
{
var
items
[]
model
.
Item
wg
:=
sync
.
WaitGroup
{}
itemsInfo
,
err
:=
getItemsInfoByKeyword
(
keyword
)
responseItems
,
err
:=
getItemsInfoByKeyword
(
keyword
)
if
err
!=
nil
{
return
nil
,
err
}
for
_
,
itemInfo
:=
range
itemsInfo
{
itemUrl
:=
itemInfo
.
Link
if
itemInfo
.
CafeName
!=
"중고나라"
{
continue
}
for
_
,
responseItem
:=
range
responseItems
{
wg
.
Add
(
1
)
go
func
(
itemUrl
string
)
{
go
func
(
responseItem
model
.
ApiResponseItem
)
{
defer
wg
.
Done
()
item
,
err
:=
crawlingNaverCafe
(
itemUrl
)
if
err
!=
nil
{
log
.
Fatal
(
err
)
}
items
=
append
(
items
,
*
item
)
}(
itemUrl
)
item
:=
model
.
Item
{
Platform
:
"중고나라"
,
Name
:
responseItem
.
Title
,
Price
:
priceStringToInt
(
responseItem
.
ProductSale
.
Cost
),
ThumbnailUrl
:
responseItem
.
ThumbnailUrl
,
ItemUrl
:
fmt
.
Sprintf
(
"https://m.cafe.naver.com/ca-fe/web/cafes/10050146/articles/%d"
,
responseItem
.
ArticleId
),
ExtraInfo
:
responseItem
.
ExtraInfo
,
}
items
=
append
(
items
,
item
)
}(
responseItem
)
}
wg
.
Wait
()
...
...
@@ -50,8 +49,8 @@ func GetItemByKeyword(keyword string) ([]model.Item, error) {
}
func
getItemsInfoByKeyword
(
keyword
string
)
([]
model
.
ApiResponseItem
,
error
)
{
encText
:=
url
.
QueryEscape
(
"중고나라 "
+
keyword
+
" 판매중"
)
apiUrl
:=
"https://openapi.naver.com/v1/search/cafearticle.json?query="
+
encText
+
"&sort=sim"
encText
:=
url
.
QueryEscape
(
keyword
)
apiUrl
:=
fmt
.
Sprintf
(
"https://apis.naver.com/cafe-web/cafe-mobile/CafeMobileWebArticleSearchListV3?cafeId=10050146&query=%s&searchBy=0&sortBy=sim&page=1&perPage=10&adUnit=MW_CAFE_BOARD"
,
encText
)
req
,
err
:=
http
.
NewRequest
(
"GET"
,
apiUrl
,
nil
)
if
err
!=
nil
{
...
...
@@ -59,6 +58,8 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) {
}
req
.
Header
.
Add
(
"X-Naver-Client-Id"
,
config
.
Cfg
.
Secret
.
CLIENTID
)
req
.
Header
.
Add
(
"X-Naver-Client-Secret"
,
config
.
Cfg
.
Secret
.
CLIENTSECRET
)
req
.
Header
.
Add
(
"Cookie"
,
config
.
Cfg
.
Header
.
Cookie
)
req
.
Header
.
Add
(
"User-agent"
,
config
.
Cfg
.
Header
.
UserAgent
)
client
:=
&
http
.
Client
{}
resp
,
err
:=
client
.
Do
(
req
)
...
...
@@ -73,55 +74,26 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) {
}(
resp
.
Body
)
response
,
_
:=
ioutil
.
ReadAll
(
resp
.
Body
)
var
apiResponse
model
.
ApiResponse
err
=
json
.
Unmarshal
(
response
,
&
apiResponse
)
if
err
!=
nil
{
log
.
Fatal
(
err
)
}
return
apiResponse
.
Items
,
nil
}
func
crawlingNaverCafe
(
cafeUrl
string
)
(
*
model
.
Item
,
error
)
{
path
,
_
:=
launcher
.
LookPath
()
u
:=
launcher
.
New
()
.
Bin
(
path
)
.
MustLaunch
()
browser
:=
rod
.
New
()
.
ControlURL
(
u
)
.
MustConnect
()
defer
func
(
browser
*
rod
.
Browser
)
{
err
:=
browser
.
Close
()
var
apiResult
map
[
string
]
interface
{}
err
=
json
.
Unmarshal
(
response
,
&
apiResult
)
if
err
!=
nil
{
log
.
Fatal
(
err
)
return
nil
,
err
}
}(
browser
)
frame
:=
browser
.
MustPage
(
cafeUrl
)
.
MustElement
(
"iframe#cafe_main"
)
time
.
Sleep
(
time
.
Second
*
2
)
source
:=
frame
.
MustFrame
()
.
MustHTML
()
html
,
err
:=
goquery
.
NewDocumentFromReader
(
bytes
.
NewReader
([]
byte
(
source
)))
result
:=
apiResult
[
"message"
]
.
(
map
[
string
]
interface
{})[
"result"
]
resultJson
,
err
:=
json
.
Marshal
(
result
)
if
err
!=
nil
{
return
nil
,
err
}
title
:=
html
.
Find
(
"h3.title_text"
)
.
Text
()
sold
:=
html
.
Find
(
"div.sold_area"
)
.
Text
()
price
:=
priceStringToInt
(
html
.
Find
(
".ProductPrice"
)
.
Text
())
thumbnailUrl
,
_
:=
html
.
Find
(
"div.product_thumb img"
)
.
Attr
(
"src"
)
extraInfo
:=
html
.
Find
(
".se-module-text"
)
.
Text
()
title
=
strings
.
TrimSpace
(
title
)
sold
=
strings
.
TrimSpace
(
sold
)
thumbnailUrl
=
strings
.
TrimSpace
(
thumbnailUrl
)
extraInfo
=
strings
.
TrimSpace
(
extraInfo
)
item
:=
model
.
Item
{
Platform
:
"중고나라"
,
Name
:
title
,
Price
:
price
,
ThumbnailUrl
:
thumbnailUrl
,
ItemUrl
:
cafeUrl
,
ExtraInfo
:
extraInfo
,
var
apiResponse
model
.
ApiResponse
err
=
json
.
Unmarshal
(
resultJson
,
&
apiResponse
)
if
err
!=
nil
{
return
nil
,
err
}
return
&
item
,
nil
return
apiResponse
.
Items
,
nil
}
func
priceStringToInt
(
priceString
string
)
int
{
...
...
Please
register
or
login
to post a comment