Use a real rate limiter to throttle Gitiles requests.
Set the default so clients of android.googlesource.com wil not get
throttled.
Change-Id: Iace07b58bc5e45343071a99133b299eca16ea045
diff --git a/cmd/slothfs-expand-manifest/main.go b/cmd/slothfs-expand-manifest/main.go
index a9ace1d..0e5dfb6 100644
--- a/cmd/slothfs-expand-manifest/main.go
+++ b/cmd/slothfs-expand-manifest/main.go
@@ -34,7 +34,13 @@
log.Fatal("must set --gitiles")
}
- service, err := gitiles.NewService(*gitilesURL)
+ // SustainedQPS is a little high, but since this is a one-shot
+ // program let's try to get away with it.
+ service, err := gitiles.NewService(*gitilesURL,
+ gitiles.Options{
+ BurstQPS: 10,
+ SustainedQPS: 5,
+ })
if err != nil {
log.Fatal(err)
}
diff --git a/cmd/slothfs-gitilesfs/main.go b/cmd/slothfs-gitilesfs/main.go
index 628b7f1..85ad920 100644
--- a/cmd/slothfs-gitilesfs/main.go
+++ b/cmd/slothfs-gitilesfs/main.go
@@ -48,7 +48,7 @@
log.Printf("NewCache: %v", err)
}
- service, err := gitiles.NewService(*url)
+ service, err := gitiles.NewService(*url, gitiles.Options{})
if err != nil {
log.Printf("NewService: %v", err)
}
diff --git a/cmd/slothfs-manifestfs/main.go b/cmd/slothfs-manifestfs/main.go
index 1b3199a..c3f57d5 100644
--- a/cmd/slothfs-manifestfs/main.go
+++ b/cmd/slothfs-manifestfs/main.go
@@ -57,7 +57,7 @@
log.Printf("NewCache: %v", err)
}
- service, err := gitiles.NewService(*gitilesURL)
+ service, err := gitiles.NewService(*gitilesURL, gitiles.Options{})
if err != nil {
log.Printf("NewService: %v", err)
}
diff --git a/cmd/slothfs-multifs/main.go b/cmd/slothfs-multifs/main.go
index 31b076b..70912b8 100644
--- a/cmd/slothfs-multifs/main.go
+++ b/cmd/slothfs-multifs/main.go
@@ -52,7 +52,7 @@
log.Printf("NewCache: %v", err)
}
- service, err := gitiles.NewService(*gitilesURL)
+ service, err := gitiles.NewService(*gitilesURL, gitiles.Options{})
if err != nil {
log.Printf("NewService: %v", err)
}
diff --git a/fs/gitilesfs_test.go b/fs/gitilesfs_test.go
index bd53d25..0bbae9d 100644
--- a/fs/gitilesfs_test.go
+++ b/fs/gitilesfs_test.go
@@ -576,7 +576,8 @@
}
fixture.service, err = gitiles.NewService(
- fmt.Sprintf("http://%s", fixture.testServer.listener.Addr().String()))
+ fmt.Sprintf("http://%s", fixture.testServer.listener.Addr().String()),
+ gitiles.Options{})
if err != nil {
return nil, err
}
diff --git a/gitiles/client.go b/gitiles/client.go
index eb600ba..10cb86a 100644
--- a/gitiles/client.go
+++ b/gitiles/client.go
@@ -25,12 +25,15 @@
"net/http"
"net/url"
"path"
+
+ "golang.org/x/net/context"
+ "golang.org/x/time/rate"
)
// Service is a client for the Gitiles JSON interface.
type Service struct {
- throttle chan struct{}
- addr url.URL
+ limiter *rate.Limiter
+ addr url.URL
}
// Addr returns the address of the gitiles service.
@@ -38,22 +41,39 @@
return s.addr.String()
}
+// Options specifies how much load we can put on remote Gitiles servers.
+type Options struct {
+ BurstQPS int
+ SustainedQPS float64
+}
+
// NewService returns a new Gitiles JSON client.
-func NewService(addr string) (*Service, error) {
+func NewService(addr string, opts Options) (*Service, error) {
+ if opts.BurstQPS == 0 {
+ opts.BurstQPS = 4
+ }
+ if opts.SustainedQPS == 0.0 {
+ opts.SustainedQPS = 0.5
+ }
+
url, err := url.Parse(addr)
if err != nil {
return nil, err
}
return &Service{
- throttle: make(chan struct{}, 12),
- addr: *url,
+ limiter: rate.NewLimiter(rate.Limit(opts.SustainedQPS), opts.BurstQPS),
+ addr: *url,
}, nil
}
func (s *Service) get(u url.URL) ([]byte, error) {
- s.throttle <- struct{}{}
+ ctx := context.Background()
+
+ if err := s.limiter.Wait(ctx); err != nil {
+ return nil, err
+ }
resp, err := http.Get(u.String())
- <-s.throttle
+
if err != nil {
return nil, err
}
