| // Copyright 2016 Google Inc. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| // This binary fetches all repos of a Gitiles host. |
| package main |
| |
| import ( |
| "bytes" |
| "encoding/json" |
| "flag" |
| "io/ioutil" |
| "log" |
| "net/http" |
| "net/url" |
| "os" |
| "path/filepath" |
| "regexp" |
| |
| "github.com/google/zoekt/gitindex" |
| ) |
| |
| func main() { |
| dest := flag.String("dest", "", "destination directory") |
| namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.") |
| excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") |
| flag.Parse() |
| |
| if len(flag.Args()) < 1 { |
| log.Fatal("must provide Gitiles URL as argument.") |
| } |
| gitilesURL, err := url.Parse(flag.Arg(0)) |
| if err != nil { |
| log.Fatal("url.Parse(): %v", err) |
| } |
| |
| if *dest == "" { |
| log.Fatal("must set --dest") |
| } |
| |
| destDir := filepath.Join(*dest, gitilesURL.Host) |
| if err := os.MkdirAll(destDir, 0755); err != nil { |
| log.Fatal(err) |
| } |
| |
| repos, err := getRepos(gitilesURL) |
| if *namePattern != "" { |
| re, err := regexp.Compile(*namePattern) |
| if err != nil { |
| log.Fatal(err) |
| } |
| |
| trimmed := map[string]string{} |
| for k, v := range repos { |
| if re.MatchString(k) { |
| trimmed[k] = v |
| } |
| } |
| repos = trimmed |
| } |
| |
| { |
| trimmed := map[string]string{} |
| for k, v := range repos { |
| if k != "All-Users" && k != "All-Projects" { |
| trimmed[k] = v |
| } |
| } |
| repos = trimmed |
| } |
| |
| if *excludePattern != "" { |
| re, err := regexp.Compile(*excludePattern) |
| if err != nil { |
| log.Fatal(err) |
| } |
| |
| trimmed := map[string]string{} |
| for k, v := range repos { |
| if !re.MatchString(k) { |
| trimmed[k] = v |
| } |
| } |
| repos = trimmed |
| } |
| if err := gitindex.CloneRepos(destDir, repos); err != nil { |
| log.Fatal(err) |
| } |
| } |
| |
| type Project struct { |
| Name string |
| CloneURL string `json:"clone_url"` |
| } |
| |
| func getRepos(URL *url.URL) (map[string]string, error) { |
| URL.RawQuery = "format=JSON" |
| resp, err := http.Get(URL.String()) |
| if err != nil { |
| return nil, err |
| } |
| defer resp.Body.Close() |
| |
| content, err := ioutil.ReadAll(resp.Body) |
| if err != nil { |
| return nil, err |
| } |
| |
| const xssTag = ")]}'\n" |
| content = bytes.TrimPrefix(content, []byte(xssTag)) |
| |
| m := map[string]*Project{} |
| if err := json.Unmarshal(content, &m); err != nil { |
| return nil, err |
| } |
| |
| result := map[string]string{} |
| for k, v := range m { |
| result[k] = v.CloneURL |
| } |
| return result, nil |
| } |