Skip to content

Commit a379535

Browse files
kubrickcodekubrickcode
committed
feat(go): libs/ratelimit 모듈 추가 및 전체 scraper에 rate limiting 적용
- libs/ratelimit 공통 모듈 생성 (Limiter 인터페이스 + 구현) - 4개 scraper 모두 ratelimit.Limiter 인터페이스로 통일 - 테스트 용이성을 위한 noopLimiter mock 패턴 적용 - 기존 golang.org/x/time/rate 직접 의존을 추상화로 개선 fix #260
1 parent 087f44b commit a379535

12 files changed

Lines changed: 715 additions & 26 deletions

File tree

‎go.work‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ use (
1212
./src/go/libs/loa-api
1313
./src/go/libs/loa-db
1414
./src/go/libs/monitoring
15+
./src/go/libs/ratelimit
1516
./src/go/libs/schedule
1617
)

‎go.work.sum‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
5353
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
5454
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
5555
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
56+
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
5657
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
5758
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
5859
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=

‎src/go/apps/auction-item-stat-scraper/scraper/scraper.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,25 @@ import (
1010
"github.com/KubrickCode/loa-work/src/go/libs/loaApi/request"
1111
"github.com/KubrickCode/loa-work/src/go/libs/loadb"
1212
"github.com/KubrickCode/loa-work/src/go/libs/loadb/models"
13-
"golang.org/x/time/rate"
13+
"github.com/KubrickCode/loa-work/src/go/libs/ratelimit"
14+
)
15+
16+
const (
17+
defaultRateLimitInterval = time.Second
18+
defaultRateLimitBurst = 1
1419
)
1520

1621
type Scraper struct {
1722
client request.APIClient
1823
db loadb.DB
19-
rateLimiter *rate.Limiter
24+
rateLimiter ratelimit.Limiter
2025
}
2126

2227
func NewScraper(client request.APIClient, db loadb.DB) *Scraper {
2328
return &Scraper{
2429
client: client,
2530
db: db,
26-
rateLimiter: rate.NewLimiter(rate.Every(time.Second), 1),
31+
rateLimiter: ratelimit.NewLimiterPerDuration(defaultRateLimitInterval, defaultRateLimitBurst),
2732
}
2833
}
2934

‎src/go/apps/market-item-category-scraper/scraper/scraper.go‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,36 @@
11
package scraper
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
67
"log"
8+
"time"
79

810
"github.com/KubrickCode/loa-work/src/go/libs/loaApi/request"
911
"github.com/KubrickCode/loa-work/src/go/libs/loadb"
1012
"github.com/KubrickCode/loa-work/src/go/libs/loadb/models"
13+
"github.com/KubrickCode/loa-work/src/go/libs/ratelimit"
14+
)
15+
16+
const (
17+
defaultRateLimitInterval = time.Second
18+
defaultRateLimitBurst = 1
1119
)
1220

1321
var ErrNoMarketItemCategories = errors.New("no market item categories found")
1422

1523
type Scraper struct {
16-
client request.APIClient
17-
db loadb.DB
24+
client request.APIClient
25+
db loadb.DB
26+
rateLimiter ratelimit.Limiter
1827
}
1928

2029
func NewScraper(client request.APIClient, db loadb.DB) *Scraper {
2130
return &Scraper{
22-
client: client,
23-
db: db,
31+
client: client,
32+
db: db,
33+
rateLimiter: ratelimit.NewLimiterPerDuration(defaultRateLimitInterval, defaultRateLimitBurst),
2434
}
2535
}
2636

@@ -41,6 +51,10 @@ func (s *Scraper) Start() error {
4151
}
4252

4353
func (s *Scraper) getCategories() ([]*models.MarketItemCategory, error) {
54+
if err := s.rateLimiter.Wait(context.Background()); err != nil {
55+
return nil, fmt.Errorf("rate limiter error: %w", err)
56+
}
57+
4458
resp, err := s.client.GetCategoryList()
4559
if err != nil {
4660
return nil, err

‎src/go/apps/market-item-category-scraper/scraper/scraper_test.go‎

Lines changed: 57 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,21 @@
11
package scraper
22

33
import (
4+
"context"
45
"errors"
56
"testing"
67

78
"github.com/KubrickCode/loa-work/src/go/libs/loaApi"
9+
"github.com/KubrickCode/loa-work/src/go/libs/ratelimit"
810
)
911

12+
// noopLimiter implements ratelimit.Limiter with no delay
13+
type noopLimiter struct{}
14+
15+
func (l *noopLimiter) Wait(ctx context.Context) error {
16+
return nil
17+
}
18+
1019
type mockAPIClient struct {
1120
getAuctionItemListFunc func(params *loaApi.GetAuctionItemListParams) (*loaApi.GetAuctionItemListResponse, error)
1221
getCategoryListFunc func() (*loaApi.GetCategoryListResponse, error)
@@ -65,8 +74,9 @@ func TestGetCategories_Success(t *testing.T) {
6574
}
6675

6776
scraper := &Scraper{
68-
client: mockClient,
69-
db: nil,
77+
client: mockClient,
78+
db: nil,
79+
rateLimiter: &noopLimiter{},
7080
}
7181

7282
categories, err := scraper.getCategories()
@@ -95,17 +105,14 @@ func TestGetCategories_APIError(t *testing.T) {
95105
}
96106

97107
scraper := &Scraper{
98-
client: mockClient,
99-
db: nil,
108+
client: mockClient,
109+
db: nil,
110+
rateLimiter: &noopLimiter{},
100111
}
101112

102113
_, err := scraper.getCategories()
103-
if err == nil {
104-
t.Fatal("Expected error, got nil")
105-
}
106-
107-
if err != expectedErr {
108-
t.Errorf("Expected error %v, got %v", expectedErr, err)
114+
if !errors.Is(err, expectedErr) {
115+
t.Fatalf("Expected error %v, got %v", expectedErr, err)
109116
}
110117
}
111118

@@ -119,8 +126,9 @@ func TestGetCategories_EmptyResponse(t *testing.T) {
119126
}
120127

121128
scraper := &Scraper{
122-
client: mockClient,
123-
db: nil,
129+
client: mockClient,
130+
db: nil,
131+
rateLimiter: &noopLimiter{},
124132
}
125133

126134
_, err := scraper.getCategories()
@@ -195,3 +203,40 @@ func TestGetFlattenCategories_EmptySubCategories(t *testing.T) {
195203
t.Errorf("Expected name 'Category without subs', got %s", flattened[0].Name)
196204
}
197205
}
206+
207+
func TestNewScraper_RateLimiterInitialization(t *testing.T) {
208+
mockClient := &mockAPIClient{}
209+
scraper := NewScraper(mockClient, nil)
210+
211+
if scraper.rateLimiter == nil {
212+
t.Fatal("Expected rateLimiter to be initialized, got nil")
213+
}
214+
}
215+
216+
func TestRateLimiter_InterfaceCompliance(t *testing.T) {
217+
mockClient := &mockAPIClient{
218+
getCategoryListFunc: func() (*loaApi.GetCategoryListResponse, error) {
219+
return &loaApi.GetCategoryListResponse{
220+
Categories: []loaApi.Category{
221+
{Code: 10000, CodeName: "Test"},
222+
},
223+
}, nil
224+
},
225+
}
226+
227+
// Test with real limiter
228+
scraper := &Scraper{
229+
client: mockClient,
230+
db: nil,
231+
rateLimiter: ratelimit.NewLimiterPerDuration(0, 1), // instant for test
232+
}
233+
234+
categories, err := scraper.getCategories()
235+
if err != nil {
236+
t.Fatalf("Expected no error, got %v", err)
237+
}
238+
239+
if len(categories) != 1 {
240+
t.Errorf("Expected 1 category, got %d", len(categories))
241+
}
242+
}

‎src/go/apps/market-item-scraper/scraper/scraper.go‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,34 @@
11
package scraper
22

33
import (
4+
"context"
45
"fmt"
56
"log"
7+
"time"
68

79
"github.com/KubrickCode/loa-work/src/go/libs/loaApi"
810
"github.com/KubrickCode/loa-work/src/go/libs/loaApi/request"
911
"github.com/KubrickCode/loa-work/src/go/libs/loadb"
1012
"github.com/KubrickCode/loa-work/src/go/libs/loadb/models"
13+
"github.com/KubrickCode/loa-work/src/go/libs/ratelimit"
14+
)
15+
16+
const (
17+
defaultRateLimitInterval = time.Second
18+
defaultRateLimitBurst = 1
1119
)
1220

1321
type Scraper struct {
14-
client request.APIClient
15-
db loadb.DB
22+
client request.APIClient
23+
db loadb.DB
24+
rateLimiter ratelimit.Limiter
1625
}
1726

1827
func NewScraper(client request.APIClient, db loadb.DB) *Scraper {
1928
return &Scraper{
20-
client: client,
21-
db: db,
29+
client: client,
30+
db: db,
31+
rateLimiter: ratelimit.NewLimiterPerDuration(defaultRateLimitInterval, defaultRateLimitBurst),
2232
}
2333
}
2434

@@ -62,6 +72,10 @@ func (s *Scraper) getItemsToSave(categories []*models.MarketItemCategory) ([]*mo
6272
pageNo := 1
6373

6474
for {
75+
if err := s.rateLimiter.Wait(context.Background()); err != nil {
76+
return nil, fmt.Errorf("rate limiter error: %w", err)
77+
}
78+
6579
resp, err := s.client.GetMarketItemList(&loaApi.GetMarketItemListParams{
6680
CategoryCode: category.Code,
6781
PageNo: pageNo,

0 commit comments

Comments
 (0)