Support HTTP proxies and cookie authentication. * Add --cookies flag, taking a cURL/Mozilla style cookie file. * Support in-memory cookie Jar in the Gitiles client. * Support redirects in the Gitiles client. * Add --agent flag to set the User-Agent string. * Add slothfs-gitiles-test to debug the whole thing. Change-Id: Ie2aec36372706f71083a323a7aa92864f2210463
diff --git a/cmd/slothfs-expand-manifest/main.go b/cmd/slothfs-expand-manifest/main.go index 2b73407..294c43c 100644 --- a/cmd/slothfs-expand-manifest/main.go +++ b/cmd/slothfs-expand-manifest/main.go
@@ -20,6 +20,7 @@ "log" "os" + "github.com/google/slothfs/cookie" "github.com/google/slothfs/gitiles" "github.com/google/slothfs/manifest" @@ -30,33 +31,46 @@ gitilesURL := flag.String("gitiles", "", "URL for gitiles") branch := flag.String("branch", "master", "branch to use for manifest") repo := flag.String("repo", "platform/manifest", "manifest repository") + cookieJarPath := flag.String("cookies", "", "path to cURL-style cookie jar file.") + agent := flag.String("agent", "slothfs-expand", "gitiles User-Agent string to use.") flag.Parse() if *gitilesURL == "" { log.Fatal("must set --gitiles") } + opts := gitiles.Options{ + UserAgent: *agent, + } + if *cookieJarPath != "" { + var err error + opts.CookieJar, err = cookie.NewJar(*cookieJarPath) + if err != nil { + log.Fatal(err) + } + } + // SustainedQPS is a little high, but since this is a one-shot // program let's try to get away with it. - service, err := gitiles.NewService(*gitilesURL, gitiles.Options{}) + service, err := gitiles.NewService(*gitilesURL, opts) if err != nil { - log.Fatal(err) + log.Fatalf("NewService: %v", err) } mf, err := fetchManifest(service, *repo, *branch) if err != nil { - log.Fatal(err) + log.Fatalf("fetchManifest: %v", err) } mf.Filter() if err := derefManifest(service, *repo, mf); err != nil { - log.Fatal(err) + log.Fatalf("derefManifest: %v", err) } xml, err := mf.MarshalXML() if err != nil { - log.Fatal(err) + log.Fatalf("MarshalXML: %v", err) } os.Stdout.Write(xml) @@ -79,7 +93,6 @@ } func derefManifest(service *gitiles.Service, manifestRepo string, mf *manifest.Manifest) error { - branchSet := map[string]struct{}{} var todoProjects []int
diff --git a/cmd/slothfs-gitiles-test/main.go b/cmd/slothfs-gitiles-test/main.go new file mode 100644 index 0000000..3e30cdc --- /dev/null +++ b/cmd/slothfs-gitiles-test/main.go
@@ -0,0 +1,129 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// slothfs-gitiles-test is a program executing a single gitiles HTTP +// request, to be used for troubleshooting proxy/auth problems. +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + "os" + "sync" + + "github.com/google/slothfs/cookie" + "github.com/google/slothfs/gitiles" +) + +func main() { + gitilesURL := flag.String("gitiles", "", "URL for gitiles") + cookieJarPath := flag.String("cookies", "", "path to cURL-style cookie jar file.") + agent := flag.String("agent", "slothfs-expand", "gitiles User-Agent string to use.") + tap := flag.Bool("tap", false, "if set, tap traffic exchanged with $http_proxy") + flag.Parse() + + if *tap { + tapTraffic() + } + if *gitilesURL == "" { + log.Fatal("must set --gitiles") + } + + opts := gitiles.Options{ + Redirect: true, + UserAgent: *agent, + } + if *cookieJarPath != "" { + var err error + opts.CookieJar, err = cookie.NewJar(*cookieJarPath) + if err != nil { + log.Fatal(err) + } + } + + service, err := gitiles.NewService(*gitilesURL, opts) + if err != nil { + log.Fatalf("NewService: %v", err) + } + + projs, err := service.List() + if err != nil { + log.Fatalf("List: %v", err) + } + + for p := range projs { + fmt.Printf("project: %s\n", p) + } +} + +func logCopy(w io.Writer, r io.Reader, who string) { + var buf [320000]byte + + for { + n, e1 := r.Read(buf[:]) + log.Println(who, string(buf[:n])) + _, e2 := w.Write(buf[:n]) + if e1 != nil || e2 != nil { + break + } + } +} + +func forward(conn net.Conn, addr string) { + f, err := net.Dial("tcp", addr) + if err != nil { + log.Fatal(err) + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + logCopy(f, conn, "A") + wg.Done() + }() + go func() { + logCopy(conn, f, "B") + wg.Done() + }() + wg.Wait() + f.Close() + conn.Close() +} + +func tapTraffic() { + proxy := os.Getenv("http_proxy") + if proxy == "" { + log.Println("no http_proxy, not tapping") + return + } + + l, err := net.Listen("tcp", ":0") + if err != nil { + log.Fatal(err) + } + os.Setenv("http_proxy", l.Addr().String()) + + go func() { + for { + c, err := l.Accept() + if err != nil { + break + } + go forward(c, proxy) + } + }() +}
diff --git a/cmd/slothfs-multifs/main.go b/cmd/slothfs-multifs/main.go index f9f9a93..1c25fcb 100644 --- a/cmd/slothfs-multifs/main.go +++ b/cmd/slothfs-multifs/main.go
@@ -23,6 +23,7 @@ "time" "github.com/google/slothfs/cache" + "github.com/google/slothfs/cookie" "github.com/google/slothfs/fs" "github.com/google/slothfs/gitiles" "github.com/hanwen/go-fuse/fuse/nodefs" @@ -33,6 +34,8 @@ cacheDir := flag.String("cache", filepath.Join(os.Getenv("HOME"), ".cache", "slothfs"), "cache dir") debug := flag.Bool("debug", false, "print debug info") config := flag.String("config", "", "JSON file configuring what repositories should be cloned.") + cookieJarPath := flag.String("cookies", "", "path to cURL-style cookie jar file.") + agent := flag.String("agent", "slothfs-multifs", "gitiles User-Agent string to use.") flag.Parse() if *cacheDir == "" { @@ -53,7 +56,18 @@ log.Printf("NewCache: %v", err) } - service, err := gitiles.NewService(*gitilesURL, gitiles.Options{}) + gitilesOpts := gitiles.Options{ + UserAgent: *agent, + } + if *cookieJarPath != "" { + var err error + gitilesOpts.CookieJar, err = cookie.NewJar(*cookieJarPath) + if err != nil { + log.Fatal(err) + } + } + + service, err := gitiles.NewService(*gitilesURL, gitilesOpts) if err != nil { log.Printf("NewService: %v", err) }
diff --git a/cookie/cookie.go b/cookie/cookie.go new file mode 100644 index 0000000..d5c8e5c --- /dev/null +++ b/cookie/cookie.go
@@ -0,0 +1,102 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// cookie parses curl cookie jar files. +package cookie + +import ( + "bufio" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "strconv" + "strings" + "time" +) + +// ParseCookieJar parses a cURL/Mozilla/Netscape cookie jar text file. +func ParseCookieJar(r io.Reader) ([]*http.Cookie, error) { + var result []*http.Cookie + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + httpOnly := false + const httpOnlyPrefix = "#HttpOnly_" + if strings.HasPrefix(line, httpOnlyPrefix) { + line = line[len(httpOnlyPrefix):] + httpOnly = true + } + if strings.HasPrefix(line, "#") { + continue + } + line = strings.TrimSpace(line) + + if line == "" { + continue + } + fields := strings.Split(line, "\t") + if len(fields) != 7 { + return nil, fmt.Errorf("got %d fields in line %q, want 8", len(fields), line) + } + + exp, err := strconv.ParseInt(fields[4], 10, 64) + if err != nil { + return nil, err + } + + c := http.Cookie{ + Domain: fields[0], + Name: fields[5], + Value: fields[6], + Path: fields[2], + Expires: time.Unix(exp, 0), + Secure: fields[3] == "TRUE", + HttpOnly: httpOnly, + } + + result = append(result, &c) + } + + return result, nil +} + +func NewJar(path string) (http.CookieJar, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + cs, err := ParseCookieJar(f) + if err != nil { + return nil, err + } + + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err + } + + for _, c := range cs { + jar.SetCookies(&url.URL{ + Scheme: "http", + Host: c.Domain, + }, []*http.Cookie{c}) + } + + return jar, nil +}
diff --git a/cookie/cookie_test.go b/cookie/cookie_test.go new file mode 100644 index 0000000..b63260e --- /dev/null +++ b/cookie/cookie_test.go
@@ -0,0 +1,52 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cookie + +import ( + "bytes" + "net/http" + "reflect" + "testing" + "time" +) + +func TestParseCookieJar(t *testing.T) { + in := `# Netscape HTTP Cookie File +# http://www.netscape.com/newsref/std/cookie_spec.html +# This is a generated file! Do not edit. +#HttpOnly_login.netscape.com FALSE / FALSE 1467968199 XYZ abc|pqr` + + buf := bytes.NewBufferString(in) + got, err := ParseCookieJar(buf) + if err != nil { + t.Fatalf("ParseCookieJar: %v", err) + } + + want := []*http.Cookie{ + { + Domain: "login.netscape.com", + Path: "/", + Secure: false, + Expires: time.Unix(1467968199, 0), + Name: "XYZ", + Value: "abc|pqr", + HttpOnly: true, + }, + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("got %#v, want %#v", got, want) + } +}
diff --git a/gitiles/client.go b/gitiles/client.go index 86845e3..2cf859f 100644 --- a/gitiles/client.go +++ b/gitiles/client.go
@@ -34,6 +34,8 @@ type Service struct { limiter *rate.Limiter addr url.URL + client http.Client + agent string } // Addr returns the address of the gitiles service. @@ -45,6 +47,12 @@ type Options struct { BurstQPS int SustainedQPS float64 + + // A writable cookie jar for storing (among others) authentication cookies. + CookieJar http.CookieJar + + // UserAgent defines how we present ourself to the server. + UserAgent string } // NewService returns a new Gitiles JSON client. @@ -60,10 +68,18 @@ if err != nil { return nil, err } - return &Service{ + s := &Service{ limiter: rate.NewLimiter(rate.Limit(opts.SustainedQPS), opts.BurstQPS), addr: *url, - }, nil + agent: opts.UserAgent, + } + + s.client.Jar = opts.CookieJar + s.client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + req.Header.Set("User-Agent", s.agent) + return nil + } + return s, nil } func (s *Service) get(u *url.URL) ([]byte, error) { @@ -72,7 +88,12 @@ if err := s.limiter.Wait(ctx); err != nil { return nil, err } - resp, err := http.Get(u.String()) + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Add("User-Agent", s.agent) + resp, err := s.client.Do(req) if err != nil { return nil, err @@ -81,6 +102,12 @@ if resp.StatusCode != 200 { return nil, fmt.Errorf("%s: %s", u.String(), resp.Status) } + if got := resp.Request.URL.String(); got != u.String() { + // We accept redirects, but only for authentication. + // If we get a 200 from a different page than we + // requested, it's probably some sort of login page. + return nil, fmt.Errorf("got URL %s, want %s", got, u.String()) + } c, err := ioutil.ReadAll(resp.Body) if err != nil {