Support HTTP proxies and cookie authentication.
* Add --cookies flag, taking a cURL/Mozilla style cookie file.
* Support in-memory cookie Jar in the Gitiles client.
* Support redirects in the Gitiles client.
* Add --agent flag to set the User-Agent string.
* Add slothfs-gitiles-test to debug the whole thing.
Change-Id: Ie2aec36372706f71083a323a7aa92864f2210463
diff --git a/cmd/slothfs-expand-manifest/main.go b/cmd/slothfs-expand-manifest/main.go
index 2b73407..294c43c 100644
--- a/cmd/slothfs-expand-manifest/main.go
+++ b/cmd/slothfs-expand-manifest/main.go
@@ -20,6 +20,7 @@
"log"
"os"
+ "github.com/google/slothfs/cookie"
"github.com/google/slothfs/gitiles"
"github.com/google/slothfs/manifest"
@@ -30,33 +31,46 @@
gitilesURL := flag.String("gitiles", "", "URL for gitiles")
branch := flag.String("branch", "master", "branch to use for manifest")
repo := flag.String("repo", "platform/manifest", "manifest repository")
+ cookieJarPath := flag.String("cookies", "", "path to cURL-style cookie jar file.")
+ agent := flag.String("agent", "slothfs-expand", "gitiles User-Agent string to use.")
flag.Parse()
if *gitilesURL == "" {
log.Fatal("must set --gitiles")
}
+ opts := gitiles.Options{
+ UserAgent: *agent,
+ }
+ if *cookieJarPath != "" {
+ var err error
+ opts.CookieJar, err = cookie.NewJar(*cookieJarPath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
// SustainedQPS is a little high, but since this is a one-shot
// program let's try to get away with it.
- service, err := gitiles.NewService(*gitilesURL, gitiles.Options{})
+ service, err := gitiles.NewService(*gitilesURL, opts)
if err != nil {
- log.Fatal(err)
+ log.Fatalf("NewService: %v", err)
}
mf, err := fetchManifest(service, *repo, *branch)
if err != nil {
- log.Fatal(err)
+ log.Fatalf("fetchManifest: %v", err)
}
mf.Filter()
if err := derefManifest(service, *repo, mf); err != nil {
- log.Fatal(err)
+ log.Fatalf("derefManifest: %v", err)
}
xml, err := mf.MarshalXML()
if err != nil {
- log.Fatal(err)
+ log.Fatalf("MarshalXML: %v", err)
}
os.Stdout.Write(xml)
@@ -79,7 +93,6 @@
}
func derefManifest(service *gitiles.Service, manifestRepo string, mf *manifest.Manifest) error {
-
branchSet := map[string]struct{}{}
var todoProjects []int
diff --git a/cmd/slothfs-gitiles-test/main.go b/cmd/slothfs-gitiles-test/main.go
new file mode 100644
index 0000000..3e30cdc
--- /dev/null
+++ b/cmd/slothfs-gitiles-test/main.go
@@ -0,0 +1,129 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// slothfs-gitiles-test is a program executing a single gitiles HTTP
+// request, to be used for troubleshooting proxy/auth problems.
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "net"
+ "os"
+ "sync"
+
+ "github.com/google/slothfs/cookie"
+ "github.com/google/slothfs/gitiles"
+)
+
+func main() {
+ gitilesURL := flag.String("gitiles", "", "URL for gitiles")
+ cookieJarPath := flag.String("cookies", "", "path to cURL-style cookie jar file.")
+ agent := flag.String("agent", "slothfs-expand", "gitiles User-Agent string to use.")
+ tap := flag.Bool("tap", false, "if set, tap traffic exchanged with $http_proxy")
+ flag.Parse()
+
+ if *tap {
+ tapTraffic()
+ }
+ if *gitilesURL == "" {
+ log.Fatal("must set --gitiles")
+ }
+
+ opts := gitiles.Options{
+ Redirect: true,
+ UserAgent: *agent,
+ }
+ if *cookieJarPath != "" {
+ var err error
+ opts.CookieJar, err = cookie.NewJar(*cookieJarPath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ service, err := gitiles.NewService(*gitilesURL, opts)
+ if err != nil {
+ log.Fatalf("NewService: %v", err)
+ }
+
+ projs, err := service.List()
+ if err != nil {
+ log.Fatalf("List: %v", err)
+ }
+
+ for p := range projs {
+ fmt.Printf("project: %s\n", p)
+ }
+}
+
+func logCopy(w io.Writer, r io.Reader, who string) {
+ var buf [320000]byte
+
+ for {
+ n, e1 := r.Read(buf[:])
+ log.Println(who, string(buf[:n]))
+ _, e2 := w.Write(buf[:n])
+ if e1 != nil || e2 != nil {
+ break
+ }
+ }
+}
+
+func forward(conn net.Conn, addr string) {
+ f, err := net.Dial("tcp", addr)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ logCopy(f, conn, "A")
+ wg.Done()
+ }()
+ go func() {
+ logCopy(conn, f, "B")
+ wg.Done()
+ }()
+ wg.Wait()
+ f.Close()
+ conn.Close()
+}
+
+func tapTraffic() {
+ proxy := os.Getenv("http_proxy")
+ if proxy == "" {
+ log.Println("no http_proxy, not tapping")
+ return
+ }
+
+ l, err := net.Listen("tcp", ":0")
+ if err != nil {
+ log.Fatal(err)
+ }
+ os.Setenv("http_proxy", l.Addr().String())
+
+ go func() {
+ for {
+ c, err := l.Accept()
+ if err != nil {
+ break
+ }
+ go forward(c, proxy)
+ }
+ }()
+}
diff --git a/cmd/slothfs-multifs/main.go b/cmd/slothfs-multifs/main.go
index f9f9a93..1c25fcb 100644
--- a/cmd/slothfs-multifs/main.go
+++ b/cmd/slothfs-multifs/main.go
@@ -23,6 +23,7 @@
"time"
"github.com/google/slothfs/cache"
+ "github.com/google/slothfs/cookie"
"github.com/google/slothfs/fs"
"github.com/google/slothfs/gitiles"
"github.com/hanwen/go-fuse/fuse/nodefs"
@@ -33,6 +34,8 @@
cacheDir := flag.String("cache", filepath.Join(os.Getenv("HOME"), ".cache", "slothfs"), "cache dir")
debug := flag.Bool("debug", false, "print debug info")
config := flag.String("config", "", "JSON file configuring what repositories should be cloned.")
+ cookieJarPath := flag.String("cookies", "", "path to cURL-style cookie jar file.")
+ agent := flag.String("agent", "slothfs-multifs", "gitiles User-Agent string to use.")
flag.Parse()
if *cacheDir == "" {
@@ -53,7 +56,18 @@
log.Printf("NewCache: %v", err)
}
- service, err := gitiles.NewService(*gitilesURL, gitiles.Options{})
+ gitilesOpts := gitiles.Options{
+ UserAgent: *agent,
+ }
+ if *cookieJarPath != "" {
+ var err error
+ gitilesOpts.CookieJar, err = cookie.NewJar(*cookieJarPath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ service, err := gitiles.NewService(*gitilesURL, gitilesOpts)
if err != nil {
log.Printf("NewService: %v", err)
}
diff --git a/cookie/cookie.go b/cookie/cookie.go
new file mode 100644
index 0000000..d5c8e5c
--- /dev/null
+++ b/cookie/cookie.go
@@ -0,0 +1,102 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// cookie parses curl cookie jar files.
+package cookie
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/cookiejar"
+ "net/url"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// ParseCookieJar parses a cURL/Mozilla/Netscape cookie jar text file.
+func ParseCookieJar(r io.Reader) ([]*http.Cookie, error) {
+ var result []*http.Cookie
+ scanner := bufio.NewScanner(r)
+ for scanner.Scan() {
+ line := scanner.Text()
+ httpOnly := false
+ const httpOnlyPrefix = "#HttpOnly_"
+ if strings.HasPrefix(line, httpOnlyPrefix) {
+ line = line[len(httpOnlyPrefix):]
+ httpOnly = true
+ }
+ if strings.HasPrefix(line, "#") {
+ continue
+ }
+ line = strings.TrimSpace(line)
+
+ if line == "" {
+ continue
+ }
+ fields := strings.Split(line, "\t")
+ if len(fields) != 7 {
+ return nil, fmt.Errorf("got %d fields in line %q, want 8", len(fields), line)
+ }
+
+ exp, err := strconv.ParseInt(fields[4], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+
+ c := http.Cookie{
+ Domain: fields[0],
+ Name: fields[5],
+ Value: fields[6],
+ Path: fields[2],
+ Expires: time.Unix(exp, 0),
+ Secure: fields[3] == "TRUE",
+ HttpOnly: httpOnly,
+ }
+
+ result = append(result, &c)
+ }
+
+ return result, nil
+}
+
+func NewJar(path string) (http.CookieJar, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ cs, err := ParseCookieJar(f)
+ if err != nil {
+ return nil, err
+ }
+
+ jar, err := cookiejar.New(nil)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, c := range cs {
+ jar.SetCookies(&url.URL{
+ Scheme: "http",
+ Host: c.Domain,
+ }, []*http.Cookie{c})
+ }
+
+ return jar, nil
+}
diff --git a/cookie/cookie_test.go b/cookie/cookie_test.go
new file mode 100644
index 0000000..b63260e
--- /dev/null
+++ b/cookie/cookie_test.go
@@ -0,0 +1,52 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cookie
+
+import (
+ "bytes"
+ "net/http"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestParseCookieJar(t *testing.T) {
+ in := `# Netscape HTTP Cookie File
+# http://www.netscape.com/newsref/std/cookie_spec.html
+# This is a generated file! Do not edit.
+#HttpOnly_login.netscape.com FALSE / FALSE 1467968199 XYZ abc|pqr`
+
+ buf := bytes.NewBufferString(in)
+ got, err := ParseCookieJar(buf)
+ if err != nil {
+ t.Fatalf("ParseCookieJar: %v", err)
+ }
+
+ want := []*http.Cookie{
+ {
+ Domain: "login.netscape.com",
+ Path: "/",
+ Secure: false,
+ Expires: time.Unix(1467968199, 0),
+ Name: "XYZ",
+ Value: "abc|pqr",
+ HttpOnly: true,
+ },
+ }
+
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("got %#v, want %#v", got, want)
+ }
+}
diff --git a/gitiles/client.go b/gitiles/client.go
index 86845e3..2cf859f 100644
--- a/gitiles/client.go
+++ b/gitiles/client.go
@@ -34,6 +34,8 @@
type Service struct {
limiter *rate.Limiter
addr url.URL
+ client http.Client
+ agent string
}
// Addr returns the address of the gitiles service.
@@ -45,6 +47,12 @@
type Options struct {
BurstQPS int
SustainedQPS float64
+
+ // A writable cookie jar for storing (among others) authentication cookies.
+ CookieJar http.CookieJar
+
+ // UserAgent defines how we present ourself to the server.
+ UserAgent string
}
// NewService returns a new Gitiles JSON client.
@@ -60,10 +68,18 @@
if err != nil {
return nil, err
}
- return &Service{
+ s := &Service{
limiter: rate.NewLimiter(rate.Limit(opts.SustainedQPS), opts.BurstQPS),
addr: *url,
- }, nil
+ agent: opts.UserAgent,
+ }
+
+ s.client.Jar = opts.CookieJar
+ s.client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+ req.Header.Set("User-Agent", s.agent)
+ return nil
+ }
+ return s, nil
}
func (s *Service) get(u *url.URL) ([]byte, error) {
@@ -72,7 +88,12 @@
if err := s.limiter.Wait(ctx); err != nil {
return nil, err
}
- resp, err := http.Get(u.String())
+ req, err := http.NewRequest("GET", u.String(), nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Add("User-Agent", s.agent)
+ resp, err := s.client.Do(req)
if err != nil {
return nil, err
@@ -81,6 +102,12 @@
if resp.StatusCode != 200 {
return nil, fmt.Errorf("%s: %s", u.String(), resp.Status)
}
+ if got := resp.Request.URL.String(); got != u.String() {
+ // We accept redirects, but only for authentication.
+ // If we get a 200 from a different page than we
+ // requested, it's probably some sort of login page.
+ return nil, fmt.Errorf("got URL %s, want %s", got, u.String())
+ }
c, err := ioutil.ReadAll(resp.Body)
if err != nil {
