| // Copyright 2016 Google Inc. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package populate |
| |
| import ( |
| "encoding/json" |
| "fmt" |
| "io" |
| "io/ioutil" |
| "log" |
| "os" |
| "path/filepath" |
| "sort" |
| "strings" |
| |
| "gopkg.in/src-d/go-git.v4/plumbing" |
| |
| "github.com/google/slothfs/gitiles" |
| "github.com/google/slothfs/manifest" |
| ) |
| |
| // fileInfo holds data files contained in the git repository within a |
| // repoTree node. |
| type fileInfo struct { |
| // the SHA1 of the file. This can be nil if getting it was too expensive. |
| sha1 *plumbing.Hash |
| } |
| |
| // repoTree is a nested set of Git repositories. |
| type repoTree struct { |
| // repositories under this repository |
| children map[string]*repoTree |
| |
| // files in this repository. |
| entries map[string]*fileInfo |
| |
| // paths that are instantiated with Copyfile or Linkfile. |
| copied []string |
| } |
| |
| // findParentRepo recursively finds the deepest child that is a prefix |
| // to the given path. |
| func (t *repoTree) findParentRepo(path string) (*repoTree, string) { |
| for k, ch := range t.children { |
| if strings.HasPrefix(path, k+"/") { |
| return ch.findParentRepo(path[len(k+"/"):]) |
| } |
| } |
| return t, path |
| } |
| |
| // write dumps the tree for debugging purposes. |
| func (t *repoTree) write(w io.Writer, indent string) { |
| for nm, ch := range t.children { |
| fmt.Fprintf(w, "%s%s:\n", indent, nm) |
| ch.write(w, indent+" ") |
| } |
| } |
| |
| // repoTreeFromManifest creates a repoTree from a manifest XML. |
| func repoTreeFromManifest(xmlFile string) (*repoTree, error) { |
| mf, err := manifest.ParseFile(xmlFile) |
| if err != nil { |
| return nil, err |
| } |
| |
| var byDepth [][]*manifest.Project |
| for i, p := range mf.Project { |
| l := len(strings.Split(p.GetPath(), "/")) |
| for len(byDepth) <= l { |
| byDepth = append(byDepth, nil) |
| } |
| |
| byDepth[l] = append(byDepth[l], &mf.Project[i]) |
| } |
| |
| root := makeRepoTree() |
| treesByPath := map[string]*repoTree{ |
| "": root, |
| } |
| |
| for _, projs := range byDepth { |
| for _, p := range projs { |
| childTree := makeRepoTree() |
| treesByPath[p.GetPath()] = childTree |
| |
| parent, key := root.findParentRepo(p.GetPath()) |
| parent.children[key] = childTree |
| } |
| } |
| |
| for _, p := range mf.Project { |
| for _, c := range p.Copyfile { |
| root.copied = append(root.copied, c.Dest) |
| } |
| for _, c := range p.Linkfile { |
| root.copied = append(root.copied, c.Dest) |
| } |
| } |
| sort.Strings(root.copied) |
| return root, nil |
| } |
| |
| // fillFromSlothFS reads tree.json to fill Entries for this repoTree |
| // node only, and does not recurse. |
| func (t *repoTree) fillFromSlothFS(dir string) error { |
| c, err := ioutil.ReadFile(filepath.Join(dir, ".slothfs", "tree.json")) |
| if err != nil { |
| return err |
| } |
| |
| var tree gitiles.Tree |
| if err := json.Unmarshal(c, &tree); err != nil { |
| return err |
| } |
| |
| for _, e := range tree.Entries { |
| fi := &fileInfo{} |
| fi.sha1, err = parseID(e.ID) |
| if err != nil { |
| return err |
| } |
| |
| t.entries[e.Name] = fi |
| } |
| |
| return nil |
| } |
| |
| // repoTreeFromSlothFS reads data from .slothfs to construct a fully |
| // populated repoTree tree. |
| func repoTreeFromSlothFS(dir string) (*repoTree, error) { |
| root, err := repoTreeFromManifest(filepath.Join(dir, ".slothfs", "manifest.xml")) |
| if err != nil { |
| return nil, err |
| } |
| |
| chs := root.allChildren() |
| errs := make(chan error, len(chs)) |
| for path, ch := range root.allChildren() { |
| go func(p string, t *repoTree) { |
| err := t.fillFromSlothFS(p) |
| errs <- err |
| }(filepath.Join(dir, path), ch) |
| } |
| |
| for i := 0; i < cap(errs); i++ { |
| err := <-errs |
| if err != nil { |
| return nil, err |
| } |
| } |
| return root, nil |
| } |
| |
| // makeRepoTree returns a repoTree struct with maps initialized. |
| func makeRepoTree() *repoTree { |
| return &repoTree{ |
| children: map[string]*repoTree{}, |
| entries: map[string]*fileInfo{}, |
| } |
| } |
| |
| // newRepoTree returns a repoTree constructed from filesystem data. |
| func newRepoTree(dir string) (*repoTree, error) { |
| t := makeRepoTree() |
| if err := t.fill(dir, ""); err != nil { |
| return nil, err |
| } |
| return t, nil |
| } |
| |
| // allChildren returns all the repositories (including the receiver) |
| // as a map keyed by relative path. |
| func (t *repoTree) allChildren() map[string]*repoTree { |
| r := map[string]*repoTree{"": t} |
| for nm, ch := range t.children { |
| for sub, subCh := range ch.allChildren() { |
| r[filepath.Join(nm, sub)] = subCh |
| } |
| } |
| return r |
| } |
| |
| // allFiles returns all the files below this repoTree. |
| func (t *repoTree) allFiles() map[string]*fileInfo { |
| r := map[string]*fileInfo{} |
| for nm, info := range t.entries { |
| r[nm] = info |
| } |
| for nm, ch := range t.children { |
| for sub, subCh := range ch.allFiles() { |
| r[filepath.Join(nm, sub)] = subCh |
| } |
| } |
| return r |
| } |
| |
| // returns whether path is the topdirectory of some git repository, |
| // either in plain git or in slothfs. |
| func isRepoDir(path string) bool { |
| if stat, err := os.Stat(filepath.Join(path, ".git")); err == nil && stat.IsDir() { |
| return true |
| } else if stat, err := os.Stat(filepath.Join(path, ".slothfs")); err == nil && stat.IsDir() { |
| return true |
| } |
| return false |
| } |
| |
| // construct fills `parent` looking through `dir` subdir of `repoRoot`. |
| func (t *repoTree) fill(repoRoot, dir string) error { |
| entries, err := ioutil.ReadDir(filepath.Join(repoRoot, dir)) |
| if err != nil { |
| log.Println(repoRoot, err) |
| return err |
| } |
| |
| todo := map[string]*repoTree{} |
| for _, e := range entries { |
| if e.IsDir() && (e.Name() == ".git" || e.Name() == ".slothfs") { |
| continue |
| } |
| if e.IsDir() && e.Name() == "out" && dir == "" { |
| // Ignore the build output directory. |
| continue |
| } |
| |
| subName := filepath.Join(dir, e.Name()) |
| if e.IsDir() { |
| if newRoot := filepath.Join(repoRoot, subName); isRepoDir(newRoot) { |
| ch := makeRepoTree() |
| t.children[subName] = ch |
| todo[newRoot] = ch |
| } else { |
| t.fill(repoRoot, subName) |
| } |
| } else { |
| t.entries[subName] = &fileInfo{} |
| } |
| } |
| |
| errs := make(chan error, len(todo)) |
| for newRoot, ch := range todo { |
| go func(r string, t *repoTree) { |
| errs <- t.fill(r, "") |
| }(newRoot, ch) |
| } |
| |
| for range todo { |
| err := <-errs |
| if err != nil { |
| return err |
| } |
| } |
| |
| return nil |
| } |