Showing
3 changed files
with
48 additions
and
66 deletions
| ... | @@ -12,6 +12,11 @@ type Config struct { | ... | @@ -12,6 +12,11 @@ type Config struct { |
| 12 | CLIENTID string `env:"SECRET.CLIENTID"` | 12 | CLIENTID string `env:"SECRET.CLIENTID"` |
| 13 | CLIENTSECRET string `env:"SECRET.CLIENTSECRET"` | 13 | CLIENTSECRET string `env:"SECRET.CLIENTSECRET"` |
| 14 | } | 14 | } |
| 15 | + | ||
| 16 | + Header struct { | ||
| 17 | + Cookie string `env:"HEADER.COOKIE"` | ||
| 18 | + UserAgent string `env:"HEADER.USERAGENT"` | ||
| 19 | + } | ||
| 15 | } | 20 | } |
| 16 | 21 | ||
| 17 | var Cfg *Config | 22 | var Cfg *Config | ... | ... |
| 1 | package model | 1 | package model |
| 2 | 2 | ||
| 3 | type ApiResponse struct { | 3 | type ApiResponse struct { |
| 4 | - LastBuildDate string `json:"lastBuildDate"` | 4 | + CafeId int `json:"cafeId"` |
| 5 | - Total uint `json:"total"` | 5 | + ArticelCount int `json:"articleCount"` |
| 6 | - Start uint `json:"start"` | 6 | + Query string `json:"query"` |
| 7 | - Display uint `json:"display"` | 7 | + Items []ApiResponseItem `json:"articleList"` |
| 8 | - Items []ApiResponseItem `json:"items"` | ||
| 9 | } | 8 | } |
| 10 | 9 | ||
| 11 | type ApiResponseItem struct { | 10 | type ApiResponseItem struct { |
| 12 | - Title string `json:"title"` | 11 | + ArticleId int `json:"articleId"` |
| 13 | - Link string `json:"link"` | 12 | + Title string `json:"subject"` |
| 14 | - Description string `json:"description"` | 13 | + ExtraInfo string `json:"summary"` |
| 15 | - CafeName string `json:"cafename"` | 14 | + ThumbnailUrl string `json:"thumbnailImageUrl"` |
| 15 | + ProductSale ApiResponseItemSale `json:"productSale"` | ||
| 16 | +} | ||
| 17 | + | ||
| 18 | +type ApiResponseItemSale struct { | ||
| 19 | + SaleStatus string `json:"saleStatue"` | ||
| 20 | + Cost string `json:"cost"` | ||
| 16 | } | 21 | } | ... | ... |
| 1 | package service | 1 | package service |
| 2 | 2 | ||
| 3 | import ( | 3 | import ( |
| 4 | - "bytes" | ||
| 5 | "encoding/json" | 4 | "encoding/json" |
| 5 | + "fmt" | ||
| 6 | "io" | 6 | "io" |
| 7 | "io/ioutil" | 7 | "io/ioutil" |
| 8 | "joongna/config" | 8 | "joongna/config" |
| ... | @@ -13,36 +13,35 @@ import ( | ... | @@ -13,36 +13,35 @@ import ( |
| 13 | "strconv" | 13 | "strconv" |
| 14 | "strings" | 14 | "strings" |
| 15 | "sync" | 15 | "sync" |
| 16 | - "time" | ||
| 17 | - | ||
| 18 | - "github.com/PuerkitoBio/goquery" | ||
| 19 | - "github.com/go-rod/rod" | ||
| 20 | - "github.com/go-rod/rod/lib/launcher" | ||
| 21 | ) | 16 | ) |
| 22 | 17 | ||
| 23 | func GetItemByKeyword(keyword string) ([]model.Item, error) { | 18 | func GetItemByKeyword(keyword string) ([]model.Item, error) { |
| 24 | var items []model.Item | 19 | var items []model.Item |
| 25 | wg := sync.WaitGroup{} | 20 | wg := sync.WaitGroup{} |
| 26 | 21 | ||
| 27 | - itemsInfo, err := getItemsInfoByKeyword(keyword) | 22 | + responseItems, err := getItemsInfoByKeyword(keyword) |
| 28 | if err != nil { | 23 | if err != nil { |
| 29 | return nil, err | 24 | return nil, err |
| 30 | } | 25 | } |
| 31 | 26 | ||
| 32 | - for _, itemInfo := range itemsInfo { | 27 | + for _, responseItem := range responseItems { |
| 33 | - itemUrl := itemInfo.Link | ||
| 34 | - if itemInfo.CafeName != "중고나라" { | ||
| 35 | - continue | ||
| 36 | - } | ||
| 37 | wg.Add(1) | 28 | wg.Add(1) |
| 38 | - go func(itemUrl string) { | 29 | + |
| 30 | + go func(responseItem model.ApiResponseItem) { | ||
| 39 | defer wg.Done() | 31 | defer wg.Done() |
| 40 | - item, err := crawlingNaverCafe(itemUrl) | ||
| 41 | if err != nil { | 32 | if err != nil { |
| 42 | log.Fatal(err) | 33 | log.Fatal(err) |
| 43 | } | 34 | } |
| 44 | - items = append(items, *item) | 35 | + item := model.Item{ |
| 45 | - }(itemUrl) | 36 | + Platform: "중고나라", |
| 37 | + Name: responseItem.Title, | ||
| 38 | + Price: priceStringToInt(responseItem.ProductSale.Cost), | ||
| 39 | + ThumbnailUrl: responseItem.ThumbnailUrl, | ||
| 40 | + ItemUrl: fmt.Sprintf("https://m.cafe.naver.com/ca-fe/web/cafes/10050146/articles/%d", responseItem.ArticleId), | ||
| 41 | + ExtraInfo: responseItem.ExtraInfo, | ||
| 42 | + } | ||
| 43 | + items = append(items, item) | ||
| 44 | + }(responseItem) | ||
| 46 | } | 45 | } |
| 47 | wg.Wait() | 46 | wg.Wait() |
| 48 | 47 | ||
| ... | @@ -50,8 +49,8 @@ func GetItemByKeyword(keyword string) ([]model.Item, error) { | ... | @@ -50,8 +49,8 @@ func GetItemByKeyword(keyword string) ([]model.Item, error) { |
| 50 | } | 49 | } |
| 51 | 50 | ||
| 52 | func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) { | 51 | func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) { |
| 53 | - encText := url.QueryEscape("중고나라 " + keyword + " 판매중") | 52 | + encText := url.QueryEscape(keyword) |
| 54 | - apiUrl := "https://openapi.naver.com/v1/search/cafearticle.json?query=" + encText + "&sort=sim" | 53 | + apiUrl := fmt.Sprintf("https://apis.naver.com/cafe-web/cafe-mobile/CafeMobileWebArticleSearchListV3?cafeId=10050146&query=%s&searchBy=0&sortBy=sim&page=1&perPage=10&adUnit=MW_CAFE_BOARD", encText) |
| 55 | 54 | ||
| 56 | req, err := http.NewRequest("GET", apiUrl, nil) | 55 | req, err := http.NewRequest("GET", apiUrl, nil) |
| 57 | if err != nil { | 56 | if err != nil { |
| ... | @@ -59,6 +58,8 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) { | ... | @@ -59,6 +58,8 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) { |
| 59 | } | 58 | } |
| 60 | req.Header.Add("X-Naver-Client-Id", config.Cfg.Secret.CLIENTID) | 59 | req.Header.Add("X-Naver-Client-Id", config.Cfg.Secret.CLIENTID) |
| 61 | req.Header.Add("X-Naver-Client-Secret", config.Cfg.Secret.CLIENTSECRET) | 60 | req.Header.Add("X-Naver-Client-Secret", config.Cfg.Secret.CLIENTSECRET) |
| 61 | + req.Header.Add("Cookie", config.Cfg.Header.Cookie) | ||
| 62 | + req.Header.Add("User-agent", config.Cfg.Header.UserAgent) | ||
| 62 | 63 | ||
| 63 | client := &http.Client{} | 64 | client := &http.Client{} |
| 64 | resp, err := client.Do(req) | 65 | resp, err := client.Do(req) |
| ... | @@ -73,55 +74,26 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) { | ... | @@ -73,55 +74,26 @@ func getItemsInfoByKeyword(keyword string) ([]model.ApiResponseItem, error) { |
| 73 | }(resp.Body) | 74 | }(resp.Body) |
| 74 | 75 | ||
| 75 | response, _ := ioutil.ReadAll(resp.Body) | 76 | response, _ := ioutil.ReadAll(resp.Body) |
| 76 | - var apiResponse model.ApiResponse | ||
| 77 | - err = json.Unmarshal(response, &apiResponse) | ||
| 78 | - if err != nil { | ||
| 79 | - log.Fatal(err) | ||
| 80 | - } | ||
| 81 | - return apiResponse.Items, nil | ||
| 82 | -} | ||
| 83 | 77 | ||
| 84 | -func crawlingNaverCafe(cafeUrl string) (*model.Item, error) { | 78 | + var apiResult map[string]interface{} |
| 85 | - path, _ := launcher.LookPath() | 79 | + err = json.Unmarshal(response, &apiResult) |
| 86 | - u := launcher.New().Bin(path).MustLaunch() | ||
| 87 | - | ||
| 88 | - browser := rod.New().ControlURL(u).MustConnect() | ||
| 89 | - defer func(browser *rod.Browser) { | ||
| 90 | - err := browser.Close() | ||
| 91 | if err != nil { | 80 | if err != nil { |
| 92 | - log.Fatal(err) | 81 | + return nil, err |
| 93 | } | 82 | } |
| 94 | - }(browser) | ||
| 95 | 83 | ||
| 96 | - frame := browser.MustPage(cafeUrl).MustElement("iframe#cafe_main") | 84 | + result := apiResult["message"].(map[string]interface{})["result"] |
| 97 | - time.Sleep(time.Second * 2) | 85 | + resultJson, err := json.Marshal(result) |
| 98 | - source := frame.MustFrame().MustHTML() | ||
| 99 | - html, err := goquery.NewDocumentFromReader(bytes.NewReader([]byte(source))) | ||
| 100 | if err != nil { | 86 | if err != nil { |
| 101 | return nil, err | 87 | return nil, err |
| 102 | } | 88 | } |
| 103 | 89 | ||
| 104 | - title := html.Find("h3.title_text").Text() | 90 | + var apiResponse model.ApiResponse |
| 105 | - sold := html.Find("div.sold_area").Text() | 91 | + err = json.Unmarshal(resultJson, &apiResponse) |
| 106 | - price := priceStringToInt(html.Find(".ProductPrice").Text()) | 92 | + if err != nil { |
| 107 | - thumbnailUrl, _ := html.Find("div.product_thumb img").Attr("src") | 93 | + return nil, err |
| 108 | - extraInfo := html.Find(".se-module-text").Text() | ||
| 109 | - | ||
| 110 | - title = strings.TrimSpace(title) | ||
| 111 | - sold = strings.TrimSpace(sold) | ||
| 112 | - thumbnailUrl = strings.TrimSpace(thumbnailUrl) | ||
| 113 | - extraInfo = strings.TrimSpace(extraInfo) | ||
| 114 | - | ||
| 115 | - item := model.Item{ | ||
| 116 | - Platform: "중고나라", | ||
| 117 | - Name: title, | ||
| 118 | - Price: price, | ||
| 119 | - ThumbnailUrl: thumbnailUrl, | ||
| 120 | - ItemUrl: cafeUrl, | ||
| 121 | - ExtraInfo: extraInfo, | ||
| 122 | } | 94 | } |
| 123 | 95 | ||
| 124 | - return &item, nil | 96 | + return apiResponse.Items, nil |
| 125 | } | 97 | } |
| 126 | 98 | ||
| 127 | func priceStringToInt(priceString string) int { | 99 | func priceStringToInt(priceString string) int { | ... | ... |
-
Please register or login to post a comment