Speed up population further. * Make .slothfs/tree.json and .slothfs/manifest.xml available * Use tree.json and manifest.xml to construct the repo tree using a few bulk-reads, rather than per-file system calls. * Do all the JSON processing in parallel across repos. * Put population code into separate package "populate" * Add an e2e test against a multifs FUSE mount. Timing (AOSP): Before: 15 secs After: 4.3 secs Change-Id: I8964a39568ff9033258e4d64be47922f06897668
diff --git a/all.bash b/all.bash index 30ba013..328a345 100644 --- a/all.bash +++ b/all.bash
@@ -16,9 +16,10 @@ for sub in manifest \ -gitiles \ -cache \ -fs \ + gitiles \ + cache \ + fs \ + populate \ cmd/slothfs-expand-manifest \ cmd/slothfs-multifs \ cmd/slothfs-manifestfs \
diff --git a/cmd/slothfs-populate/main.go b/cmd/slothfs-populate/main.go index 2c67fa1..5c50d79 100644 --- a/cmd/slothfs-populate/main.go +++ b/cmd/slothfs-populate/main.go
@@ -15,406 +15,14 @@ package main import ( - "bytes" "flag" - "fmt" - "io/ioutil" "log" "os" - "path/filepath" - "sort" - "strings" - "syscall" "time" - git "github.com/libgit2/git2go" + "github.com/google/slothfs/populate" ) -type fileInfo struct { - isRegular bool - size int64 - inode uint64 -} - -type repoTree struct { - // repositories under this repository - children map[string]*repoTree - - // files in this repository. - entries map[string]*fileInfo -} - -func makeRepoTree() *repoTree { - return &repoTree{ - children: map[string]*repoTree{}, - entries: map[string]*fileInfo{}, - } -} - -func newRepoTree(dir string) (*repoTree, error) { - t := makeRepoTree() - if err := t.fill(dir, ""); err != nil { - return nil, err - } - return t, nil -} - -// allChildren returns all the repositories (including the receiver) -// as a map keyed by relative path. -func (t *repoTree) allChildren() map[string]*repoTree { - r := map[string]*repoTree{"": t} - for nm, ch := range t.children { - for sub, subCh := range ch.allChildren() { - r[filepath.Join(nm, sub)] = subCh - } - } - return r -} - -// allFiles returns all the files below this repoTree. -func (t *repoTree) allFiles() map[string]*fileInfo { - r := map[string]*fileInfo{} - for nm, info := range t.entries { - r[nm] = info - } - for nm, ch := range t.children { - for sub, subCh := range ch.allFiles() { - r[filepath.Join(nm, sub)] = subCh - } - } - return r -} - -func isRepoDir(path string) bool { - if stat, err := os.Stat(filepath.Join(path, ".git")); err == nil && stat.IsDir() { - return true - } else if stat, err := os.Stat(filepath.Join(path, ".gitid")); err == nil && !stat.IsDir() { - return true - } - return false -} - -// construct fills `parent` looking through `dir` subdir of `repoRoot`. -func (parent *repoTree) fill(repoRoot, dir string) error { - entries, err := ioutil.ReadDir(filepath.Join(repoRoot, dir)) - if err != nil { - return err - } - - todo := map[string]*repoTree{} - for _, e := range entries { - if (e.IsDir() && e.Name() == ".git") || (!e.IsDir() && e.Name() == ".gitid") { - continue - } - if e.IsDir() && e.Name() == "out" && dir == "" { - // Ignore the build output directory. - continue - } - - subName := filepath.Join(dir, e.Name()) - if e.IsDir() { - if newRoot := filepath.Join(repoRoot, subName); isRepoDir(newRoot) { - ch := makeRepoTree() - parent.children[subName] = ch - todo[newRoot] = ch - } else { - parent.fill(repoRoot, subName) - } - } else { - parent.entries[subName] = &fileInfo{ - isRegular: e.Mode()&os.ModeType == 0, - size: e.Size(), - inode: e.Sys().(*syscall.Stat_t).Ino, - } - } - } - - errs := make(chan error, len(todo)) - for newRoot, ch := range todo { - go func(r string, t *repoTree) { - errs <- t.fill(r, "") - }(newRoot, ch) - } - - for range todo { - err := <-errs - if err != nil { - return err - } - } - - return nil -} - -// symlinkRepo creates symlinks for all the files in `child`. -func symlinkRepo(name string, child *repoTree, roRoot, rwRoot string) error { - fi, err := os.Stat(filepath.Join(rwRoot, name)) - if err == nil && fi.IsDir() { - return nil - } - - for e := range child.entries { - dest := filepath.Join(rwRoot, name, e) - - if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { - return err - } - if err := os.Symlink(filepath.Join(roRoot, name, e), dest); err != nil { - return err - } - } - return nil -} - -// createTreeLinks tries to short-cut symlinks for whole trees by -// symlinking to the root of a repository in the RO tree. -func createTreeLinks(ro, rw *repoTree, roRoot, rwRoot string) error { - allRW := rw.allChildren() - -outer: - for nm, ch := range ro.children { - foundCheckout := false - foundRecurse := false - for k := range allRW { - if k == "" { - continue - } - if nm == k { - foundRecurse = true - break - } - rel, err := filepath.Rel(nm, k) - if err != nil { - return err - } - - if strings.HasPrefix(rel, "..") { - continue - } - - // we have a checkout below "nm". - foundCheckout = true - break - } - - switch { - case foundRecurse: - if err := createTreeLinks(ch, rw.children[nm], filepath.Join(roRoot, nm), filepath.Join(rwRoot, nm)); err != nil { - return err - } - continue outer - case !foundCheckout: - dest := filepath.Join(rwRoot, nm) - if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { - return err - } - if err := os.Symlink(filepath.Join(roRoot, nm), dest); err != nil { - return err - } - } - } - return nil -} - -// createLinks will populate a RW tree with symlinks to the RO tree. -func createLinks(ro, rw *repoTree, roRoot, rwRoot string) error { - if err := createTreeLinks(ro, rw, roRoot, rwRoot); err != nil { - return err - } - - rwc := rw.allChildren() - for nm, ch := range ro.allChildren() { - if _, ok := rwc[nm]; !ok { - if err := symlinkRepo(nm, ch, roRoot, rwRoot); err != nil { - return err - } - } - } - return nil -} - -// clearLinks removes all symlinks to the RO tree. It returns the workspace name that was linked before. -func clearLinks(mount, dir string) (string, error) { - mount = filepath.Clean(mount) - - var prefix string - var dirs []string - if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error { - if fi.Mode()&os.ModeSymlink != 0 { - target, err := os.Readlink(n) - if err != nil { - return err - } - if strings.HasPrefix(target, mount) { - prefix = target - if err := os.Remove(n); err != nil { - return err - } - } - } - if fi.IsDir() { - dirs = append(dirs, n) - } - return nil - }); err != nil { - return "", err - } - - // Reverse the ordering, so we get the deepest subdirs first. - sort.Strings(dirs) - for i := range dirs { - d := dirs[len(dirs)-1-i] - // Ignore error: dir may still contain entries. - os.Remove(d) - } - - prefix = strings.TrimPrefix(prefix, mount+"/") - if i := strings.Index(prefix, "/"); i != -1 { - prefix = prefix[:i] - } - return prefix, nil -} - -const attrName = "user.gitsha1" - -func getSHA1(fn string) (*git.Oid, error) { - var data [40]byte - sz, err := syscall.Getxattr(fn, attrName, data[:]) - if err != nil { - return nil, fmt.Errorf("Getxattr(%s, %s): %v", fn, attrName, err) - } - - oid, err := git.NewOid(string(data[:sz])) - if err != nil { - return nil, err - } - return oid, nil -} - -// Returns the filenames (as relative paths) in newDir that have -// changed relative to the files in oldDir. -func changedFiles(oldDir string, oldInfos map[string]*fileInfo, - newDir string, newInfos map[string]*fileInfo) ([]string, error) { - var changed []string - for path, info := range newInfos { - if path == "manifest.xml" { - continue - } - - if !info.isRegular { - // TODO(hanwen): this is incorrect. If a file - // changes from a blob to a symlink, we should - // deref the symlink and check if the blob has - // changed. - continue - } - - old, ok := oldInfos[path] - if !ok { - changed = append(changed, path) - continue - } - - if old.inode == info.inode { - continue - } - - if old.size != info.size { - changed = append(changed, path) - continue - } - - oldSHA1, err := getSHA1(filepath.Join(oldDir, path)) - if err != nil { - return nil, err - } - newSHA1, err := getSHA1(filepath.Join(newDir, path)) - if err != nil { - return nil, err - } - - if bytes.Compare(oldSHA1[:], newSHA1[:]) != 0 { - changed = append(changed, path) - } - } - - sort.Strings(changed) - return changed, nil -} - -// populateCheckout updates a RW dir with new symlinks to the given RO dir. -func populateCheckout(ro, rw string) error { - ro = filepath.Clean(ro) - wsName, err := clearLinks(filepath.Dir(ro), rw) - if err != nil { - return err - } - oldRoot := filepath.Join(filepath.Dir(ro), wsName) - - // Do the file system traversals in parallel. - errs := make(chan error, 3) - var rwTree, roTree *repoTree - var oldInfos map[string]*fileInfo - - if wsName != "" { - go func() { - t, err := newRepoTree(oldRoot) - oldInfos = t.allFiles() - errs <- err - }() - } else { - oldInfos = map[string]*fileInfo{} - errs <- nil - } - - go func() { - t, err := newRepoTree(rw) - rwTree = t - errs <- err - }() - go func() { - t, err := newRepoTree(ro) - roTree = t - errs <- err - }() - - for i := 0; i < cap(errs); i++ { - err := <-errs - if err != nil { - return err - } - } - - if err := createLinks(roTree, rwTree, ro, rw); err != nil { - return err - } - - changed, err := changedFiles(oldRoot, oldInfos, ro, roTree.allFiles()) - if err != nil { - return fmt.Errorf("changedFiles: %v", err) - } - - for i, p := range changed { - changed[i] = filepath.Join(ro, p) - } - - if err := seqTouch(changed, time.Now()); err != nil { - return err - } - - return nil -} - -func seqTouch(fs []string, t time.Time) error { - for _, f := range fs { - if err := os.Chtimes(f, t, t); err != nil { - return err - } - } - - return nil -} - func main() { mount := flag.String("ro", "", "path to slothfs-multifs mount.") flag.Parse() @@ -426,7 +34,16 @@ log.Fatal("too many arguments.") } - if err := populateCheckout(*mount, dir); err != nil { - log.Fatalf("populateCheckout: %v", err) + changed, err := populate.Checkout(*mount, dir) + if err != nil { + log.Fatalf("populate.Checkout: %v", err) } + + now := time.Now() + for _, c := range changed { + if err := os.Chtimes(c, now, now); err != nil { + log.Fatalf("Chtimes(%s): %v", c, err) + } + } + log.Printf("touched %d files", len(changed)) }
diff --git a/cmd/slothfs-populate/main_test.go b/cmd/slothfs-populate/main_test.go deleted file mode 100644 index 91d88a5..0000000 --- a/cmd/slothfs-populate/main_test.go +++ /dev/null
@@ -1,259 +0,0 @@ -// Copyright 2016 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "syscall" - "testing" -) - -const attr = "user.gitsha1" -const checksum = "3f75526aa8f01eea5d76cee10722195dc73676de" - -func createFSTree(names []string) (string, error) { - dir, err := ioutil.TempDir("", "") - if err != nil { - return dir, err - } - - for _, f := range names { - p := filepath.Join(dir, f) - if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { - return dir, err - } - if err := ioutil.WriteFile(p, []byte{42}, 0644); err != nil { - return dir, err - } - if err := syscall.Setxattr(p, attr, []byte(checksum), 0); err != nil { - return dir, fmt.Errorf("Setxattr: %v", err) - } - } - return dir, nil -} - -func TestConstruct(t *testing.T) { - dir, err := createFSTree([]string{ - "toplevel", - "build/core/.git/HEAD", - "build/core/subdir/core.h", - "build/core/song/.git/HEAD", - "build/core/song/song.mp3", - "build/core/top", - "build/subfile", - }) - if err != nil { - t.Fatal(err) - } - songT := &repoTree{ - children: map[string]*repoTree{}, - entries: map[string]*fileInfo{ - "song.mp3": &fileInfo{isRegular: true, size: 1}, - }, - } - coreT := &repoTree{ - children: map[string]*repoTree{"song": songT}, - entries: map[string]*fileInfo{ - "subdir/core.h": &fileInfo{isRegular: true, size: 1}, - "top": &fileInfo{isRegular: true, size: 1}, - }, - } - topT := &repoTree{ - children: map[string]*repoTree{ - "build/core": coreT, - }, - entries: map[string]*fileInfo{ - "build/subfile": &fileInfo{isRegular: true, size: 1}, - "toplevel": &fileInfo{ - isRegular: true, size: 1}, - }, - } - - got, err := newRepoTree(dir) - if err != nil { - t.Fatalf("newRepoTree: %v", err) - } - - // Clear unpredictable data. - gotCh := got.allChildren() - for _, t := range gotCh { - for _, e := range t.entries { - e.inode = 0 - } - } - - wantCh := topT.allChildren() - for k, v := range wantCh { - if !reflect.DeepEqual(v, gotCh[k]) { - t.Fatalf("subrepo %q: got %#v want %#v", v, gotCh[k]) - } - } - - if !reflect.DeepEqual(got, topT) { - t.Errorf("got %#v want %#v", got, topT) - } -} - -func TestPopulate(t *testing.T) { - dir, err := createFSTree([]string{ - "ro/toplevel", - "ro/build/core/.gitid", - "ro/build/core/subdir/core.h", - "ro/build/core/song/.gitid", - "ro/build/core/song/song.mp3", - "ro/build/core/top", - "ro/build/subfile", - "ro/platform/art/.gitid", - "ro/platform/art/art.c", - "ro/platform/art/art.h", - "ro/platform/art/painting/.gitid", - "ro/platform/art/painting/picasso.c", - "rw/build/core/.git/head", - "rw/build/core/newdir/bla", - }) - if err != nil { - t.Fatal("createFSTree:", err) - } - - if err := os.Symlink(filepath.Join(dir, "ro/obsolete"), filepath.Join(dir, "rw/obsolete")); err != nil { - t.Errorf("Symlink: %v", err) - } - - if err := populateCheckout(filepath.Join(dir, "ro"), filepath.Join(dir, "rw")); err != nil { - t.Errorf("populateCheckout: %v", err) - } - - for _, f := range []string{ - "build/core/newdir/bla", - "build/core/song/song.mp3", - "platform/art/art.c", - "platform/art/art.h", - "platform/art/painting/picasso.c", - } { - fn := filepath.Join(dir, "rw", f) - fi, err := os.Stat(fn) - if err != nil || fi.Size() != 1 { - t.Errorf("Stat(%s): %v, %v", fn, fi, err) - } - } - - // The following files are in repo that has a r/w checkout, so - // they should not appear in the populated tree. - for _, f := range []string{ - "ro/build/core/subdir/core.h", - "ro/build/core/top", - } { - fn := filepath.Join(dir, "rw", f) - _, err := os.Stat(fn) - if err == nil { - t.Errorf("file %s exists", fn) - } - } - - if fi, err := os.Lstat(filepath.Join(dir, "rw/obsolete")); err == nil { - t.Fatalf("obsolete symlink still there: %v", fi) - } -} - -func TestChangedFiles(t *testing.T) { - dir, err := createFSTree([]string{ - "r1/manifest.xml", - "r1/same", - "r1/checksum", - "r1/size", - "r2/manifest.xml", - "r2/same", - "r2/checksum", - "r2/newfile", - "r2/size", - }) - if err != nil { - t.Fatalf("createFSTree: %v", err) - } - - if err := os.Symlink("same", filepath.Join(dir, "r2/symlink")); err != nil { - t.Fatalf("symlink: %v", err) - } - - // same size, different checksum. - ck2 := "3f75526aa8f01eea5d76cee10722195dc73676df" - if err := syscall.Setxattr(filepath.Join(dir, "r2/checksum"), attr, []byte(ck2), 0); err != nil { - t.Fatalf("Setxattr: %v", err) - } - - // different size. - if err := ioutil.WriteFile(filepath.Join(dir, "r2/size"), []byte("changed"), 0644); err != nil { - t.Fatalf("WriteFile(%s/r2/size): %v", dir, err) - } - - // Manifest should be ignored. - if err := ioutil.WriteFile(filepath.Join(dir, "r2/manifest.xml"), []byte("changed"), 0644); err != nil { - t.Fatalf("WriteFile(%s/r2/manifest): %v", dir, err) - } - - r2tree, err := newRepoTree(filepath.Join(dir, "r2")) - if err != nil { - t.Fatalf("newRepoTree: %v", err) - } - r1tree, err := newRepoTree(filepath.Join(dir, "r1")) - if err != nil { - t.Fatalf("newRepoTree: %v", err) - } - - oldRoot := filepath.Join(dir, "r1") - got, err := changedFiles(oldRoot, r1tree.allFiles(), filepath.Join(dir, "r2"), r2tree.allFiles()) - if err != nil { - t.Fatalf("changedFiles: %v", err) - } - if want := []string{"checksum", "newfile", "size"}; !reflect.DeepEqual(want, got) { - t.Errorf("got %v, want %v", got, want) - } -} - -func TestClearEmptyDirs(t *testing.T) { - dir, err := createFSTree([]string{ - "ro/build/sub/sub2/p1/.gitid", - "ro/build/sub/sub2/p1/build.mk", - - "rw/build/proj/.git/HEAD", - "rw/build/proj/build.mk", - - "r3/toplevel", - }) - if err != nil { - t.Fatal("createFSTree:", err) - } - - dest := filepath.Join(dir, "rw", "build/sub/sub2/p1") - if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { - t.Errorf("MkdirAll: %v", err) - } - if err := os.Symlink(filepath.Join(dir, "ro", "build/sub/sub2/p1"), dest); err != nil { - t.Errorf("Symlink(%s): %v", dest, err) - } - - if err := populateCheckout(filepath.Join(dir, "r3"), filepath.Join(dir, "rw")); err != nil { - t.Errorf("populateCheckout: %v", err) - } - - gone := filepath.Join(dir, "rw", "build", "sub") - if fi, err := os.Lstat(gone); err == nil { - t.Errorf("directory %s still there: %v", gone, fi) - } -}
diff --git a/fs/gitilesfs.go b/fs/gitilesfs.go index 7936604..de4661c 100644 --- a/fs/gitilesfs.go +++ b/fs/gitilesfs.go
@@ -15,6 +15,7 @@ package fs import ( + "encoding/json" "fmt" "io" "log" @@ -410,8 +411,15 @@ } - r.Inode().NewChild(".gitid", - false, newDataNode([]byte(r.tree.ID))) + slothfsNode := r.Inode().NewChild(".slothfs", true, newDirNode()) + slothfsNode.NewChild("treeID", false, newDataNode([]byte(r.tree.ID))) + + treeContent, err := json.MarshalIndent(r.tree, "", " ") + if err != nil { + log.Panicf("json.Marshal: %v", err) + } + + slothfsNode.NewChild("tree.json", false, newDataNode([]byte(treeContent))) // We don't need the tree data anymore. r.tree = nil
diff --git a/fs/gitilesfs_test.go b/fs/gitilesfs_test.go index 78f75ea..eaddca9 100644 --- a/fs/gitilesfs_test.go +++ b/fs/gitilesfs_test.go
@@ -271,9 +271,9 @@ } want := "58d9fdae2c26d82e04f3fcafc4358b99109f0e70" - path := filepath.Join(fix.mntDir, ".gitid") + path := filepath.Join(fix.mntDir, ".slothfs/treeID") if got, err := ioutil.ReadFile(path); err != nil { - t.Errorf("ReadFile(.gitid): %v", err) + t.Errorf("ReadFile(.slothfs/treeID): %v", err) } else if string(got) != want { t.Errorf("got %q, want %q", got, want) } @@ -561,7 +561,7 @@ } defer fix.cleanup() - xmlPath := filepath.Join(fix.mntDir, "manifest.xml") + xmlPath := filepath.Join(fix.mntDir, ".slothfs", "manifest.xml") fuseMF, err := manifest.ParseFile(xmlPath) if err != nil { t.Fatalf("ParseFile(%s): %v", xmlPath, err)
diff --git a/fs/manifestfs.go b/fs/manifestfs.go index 9a51150..888eda7 100644 --- a/fs/manifestfs.go +++ b/fs/manifestfs.go
@@ -15,6 +15,7 @@ package fs import ( + "encoding/json" "fmt" "log" "path/filepath" @@ -199,7 +200,15 @@ } } - fs.Inode().NewChild("manifest.xml", false, newDataNode(fs.manifestXML)) + metaNode := fs.Inode().NewChild(".slothfs", true, newDirNode()) + metaNode.NewChild("manifest.xml", false, newDataNode(fs.manifestXML)) + + var tree gitiles.Tree + treeContent, err := json.Marshal(tree) + if err != nil { + log.Panicf("json.Marshal: %v", err) + } + metaNode.NewChild("tree.json", false, newDataNode(treeContent)) return nil }
diff --git a/populate/e2e_test.go b/populate/e2e_test.go new file mode 100644 index 0000000..47223ff --- /dev/null +++ b/populate/e2e_test.go
@@ -0,0 +1,246 @@ +package populate + +import ( + "fmt" + "io/ioutil" + "log" + "net" + "os" + "path/filepath" + "testing" + + "github.com/google/slothfs/cache" + "github.com/google/slothfs/fs" + "github.com/google/slothfs/gitiles" + "github.com/google/slothfs/manifest" + "github.com/hanwen/go-fuse/fuse/nodefs" + + git "github.com/libgit2/git2go" +) + +// a bunch of random sha1s. +var ids = []string{ + "f065f1478dc8bfebdc59f20fb2fc1f8da4d7c334", + "ae6d11c113a0a20be662df287899046f74092abe", + "9200e4a97b6e051dd56d3de5378febae40a367e9", + "7ba00d0407ed4467c874ab45bb47fcb82fe63fac", +} + +func gitID(s string) *git.Oid { + i, err := git.NewOid(s) + if err != nil { + log.Panicf("NewOid(%q): %v", i, err) + } + return i +} + +func newInt(i int) *int { + return &i +} + +func abortListener(l net.Listener) { + _, err := l.Accept() + if err == nil { + log.Panicf("got incoming connection") + } +} + +func TestFUSE(t *testing.T) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + for _, d := range []string{"mnt", "ws", "cache"} { + if err := os.MkdirAll(filepath.Join(dir, d), 0755); err != nil { + t.Fatal(err) + } + } + + cache, err := cache.NewCache(filepath.Join(dir, "cache"), cache.Options{}) + if err != nil { + t.Fatal(err) + } + + // Setup a fake gitiles; make sure we never talk to it. + l, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + go abortListener(l) + defer l.Close() + + service, err := gitiles.NewService(fmt.Sprintf("http://%s/", l.Addr()), gitiles.Options{}) + if err != nil { + log.Printf("NewService: %v", err) + } + + opts := fs.MultiFSOptions{} + + root := fs.NewMultiFS(service, cache, opts) + fuseOpts := nodefs.NewOptions() + server, _, err := nodefs.MountRoot(filepath.Join(dir, "mnt"), root, fuseOpts) + if err != nil { + t.Fatal(err) + } + + go server.Serve() + defer server.Unmount() + + // We avoid talking to gitiles by inserting entries into the + // cache manually. + if err := cache.Tree.Add(gitID(ids[0]), &gitiles.Tree{ + ID: ids[0], + Entries: []gitiles.TreeEntry{ + { + Mode: 0100644, + Name: "a", + Type: "blob", + ID: ids[1], + Size: newInt(42), + }, + { + Mode: 0100644, + Name: "b/c", + Type: "blob", + ID: ids[2], + Size: newInt(1), + }, + }, + }); err != nil { + t.Fatal(err) + } + if err := cache.Tree.Add(gitID(ids[1]), &gitiles.Tree{ + ID: ids[1], + Entries: []gitiles.TreeEntry{ + { + Mode: 0100644, + Name: "a", + Type: "blob", + ID: ids[2], + Size: newInt(42), + }, + { + Mode: 0100644, + Name: "b/c", + Type: "blob", + ID: ids[2], + Size: newInt(1), + }, + { + Mode: 0100644, + Name: "new", + Type: "blob", + ID: ids[3], + Size: newInt(42), + }, + }, + }); err != nil { + t.Fatal(err) + } + if err := cache.Tree.Add(gitID(ids[2]), &gitiles.Tree{ + ID: ids[2], + Entries: []gitiles.TreeEntry{ + { + Mode: 0100644, + Name: "d", + Type: "blob", + ID: ids[3], + Size: newInt(42), + }, + }, + }); err != nil { + t.Fatal(err) + } + + mf1 := manifest.Manifest{ + Project: []manifest.Project{{ + Name: "platform/project", + Path: "project", + Revision: ids[0], + }}} + + mf2 := manifest.Manifest{ + Project: []manifest.Project{ + { + Name: "platform/project", + Path: "project", + Revision: ids[1], + }, { + Name: "platform/sub", + Path: "sub", + Revision: ids[2], + }}, + } + + bytes1, err := mf1.MarshalXML() + if err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(dir, "m1.xml"), bytes1, 0644); err != nil { + t.Fatal(err) + } + + bytes2, err := mf2.MarshalXML() + if err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(dir, "m2.xml"), bytes2, 0644); err != nil { + t.Fatal(err) + } + + if err := os.Symlink(filepath.Join(dir, "m1.xml"), filepath.Join(dir, "mnt", "config", "m1")); err != nil { + t.Fatal(err) + } + + testFile := filepath.Join(dir, "mnt", "m1", "project", "b/c") + if fi, err := os.Lstat(testFile); err != nil { + t.Fatalf("Lstat(%s): %v", testFile, err) + } else if fi.Size() != 1 { + t.Fatalf("%s has size %d", fi.Size()) + } + + if err := os.Symlink(filepath.Join(dir, "m2.xml"), filepath.Join(dir, "mnt", "config", "m2")); err != nil { + t.Fatal(err) + } + + ws := filepath.Join(dir, "ws") + + if _, err := Checkout(filepath.Join(dir, "mnt", "m1"), ws); err != nil { + t.Fatal("Checkout m1:", err) + } + + if dest, err := os.Readlink(filepath.Join(ws, "project")); err != nil { + t.Fatal(err) + } else if want := filepath.Join(dir, "mnt", "m1", "project"); dest != want { + t.Fatalf("got %q, want %q", dest, want) + } + + // Make sure we detect changed files. We have to be careful in + // the test setup that no blobs are shared with newly + // appearing files, or they'll be touched for being new files. + + changed, err := Checkout(filepath.Join(dir, "mnt", "m2"), ws) + if err != nil { + t.Fatal(err) + } + + for _, f := range []string{"project/a", "project/new"} { + found := false + for _, c := range changed { + if c == filepath.Join(dir, "mnt", "m2", f) { + found = true + break + } + } + if !found { + t.Errorf("file %s was not changed.", f) + } + } + + if dest, err := os.Readlink(filepath.Join(ws, "sub")); err != nil { + t.Fatal(err) + } else if want := filepath.Join(dir, "mnt", "m2", "sub"); dest != want { + t.Fatalf("got %q, want %q", dest, want) + } +}
diff --git a/populate/populate.go b/populate/populate.go new file mode 100644 index 0000000..e8b91a3 --- /dev/null +++ b/populate/populate.go
@@ -0,0 +1,249 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// populate holds the code to augment a partial R/W checkout with a +// symlink forest into a SlothFS workspace. +package populate + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// symlinkRepo creates symlinks for all the files in `child`. +func symlinkRepo(name string, child *repoTree, roRoot, rwRoot string) error { + fi, err := os.Stat(filepath.Join(rwRoot, name)) + if err == nil && fi.IsDir() { + return nil + } + + for e := range child.entries { + dest := filepath.Join(rwRoot, name, e) + + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return err + } + if err := os.Symlink(filepath.Join(roRoot, name, e), dest); err != nil { + return err + } + } + return nil +} + +// createTreeLinks tries to short-cut symlinks for whole trees by +// symlinking to the root of a repository in the RO tree. +func createTreeLinks(ro, rw *repoTree, roRoot, rwRoot string) error { + allRW := rw.allChildren() + +outer: + for nm, ch := range ro.children { + foundCheckout := false + foundRecurse := false + for k := range allRW { + if k == "" { + continue + } + if nm == k { + foundRecurse = true + break + } + rel, err := filepath.Rel(nm, k) + if err != nil { + return err + } + + if strings.HasPrefix(rel, "..") { + continue + } + + // we have a checkout below "nm". + foundCheckout = true + break + } + + switch { + case foundRecurse: + if err := createTreeLinks(ch, rw.children[nm], filepath.Join(roRoot, nm), filepath.Join(rwRoot, nm)); err != nil { + return err + } + continue outer + case !foundCheckout: + dest := filepath.Join(rwRoot, nm) + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return err + } + if err := os.Symlink(filepath.Join(roRoot, nm), dest); err != nil { + return err + } + } + } + return nil +} + +// createLinks will populate a RW tree with symlinks to the RO tree. +func createLinks(ro, rw *repoTree, roRoot, rwRoot string) error { + if err := createTreeLinks(ro, rw, roRoot, rwRoot); err != nil { + return err + } + + rwc := rw.allChildren() + for nm, ch := range ro.allChildren() { + if _, ok := rwc[nm]; !ok { + if err := symlinkRepo(nm, ch, roRoot, rwRoot); err != nil { + return err + } + } + } + return nil +} + +// clearLinks removes all symlinks to the RO tree. It returns the workspace name that was linked before. +func clearLinks(mount, dir string) (string, error) { + mount = filepath.Clean(mount) + + var prefix string + var dirs []string + if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error { + if fi == nil { + return fmt.Errorf("Walk %s: nil fileinfo for %s", dir, n) + } + if fi.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(n) + if err != nil { + return err + } + if strings.HasPrefix(target, mount) { + prefix = target + if err := os.Remove(n); err != nil { + return err + } + } + } + if fi.IsDir() && n != dir { + dirs = append(dirs, n) + } + return nil + }); err != nil { + return "", fmt.Errorf("Walk %s: %v", dir, err) + } + + // Reverse the ordering, so we get the deepest subdirs first. + sort.Strings(dirs) + for i := range dirs { + d := dirs[len(dirs)-1-i] + // Ignore error: dir may still contain entries. + os.Remove(d) + } + + prefix = strings.TrimPrefix(prefix, mount+"/") + if i := strings.Index(prefix, "/"); i != -1 { + prefix = prefix[:i] + } + return prefix, nil +} + +// Returns the filenames (as relative paths) in newDir that have +// changed relative to the files in oldDir. +func changedFiles(oldInfos map[string]*fileInfo, newInfos map[string]*fileInfo) ([]string, error) { + var changed []string + for path, info := range newInfos { + old, ok := oldInfos[path] + if !ok { + changed = append(changed, path) + continue + } + if info.isLink { + // TODO(hanwen): maybe we should we deref the link? + continue + } + + if old.sha1 == nil || info.sha1 == nil { + changed = append(changed, path) + continue + } + if bytes.Compare(old.sha1[:], info.sha1[:]) != 0 { + changed = append(changed, path) + continue + } + } + sort.Strings(changed) + return changed, nil +} + +// Checkout updates a RW dir with new symlinks to the given RO dir. +// Returns the files that should be touched. +func Checkout(ro, rw string) ([]string, error) { + ro = filepath.Clean(ro) + wsName, err := clearLinks(filepath.Dir(ro), rw) + if err != nil { + return nil, err + } + oldRoot := filepath.Join(filepath.Dir(ro), wsName) + + // Do the file system traversals in parallel. + errs := make(chan error, 3) + var rwTree, roTree *repoTree + var oldInfos map[string]*fileInfo + + if wsName != "" { + go func() { + t, err := repoTreeFromSlothFS(oldRoot) + if t != nil { + oldInfos = t.allFiles() + } + errs <- err + }() + } else { + oldInfos = map[string]*fileInfo{} + errs <- nil + } + + go func() { + t, err := newRepoTree(rw) + rwTree = t + errs <- err + }() + go func() { + t, err := repoTreeFromSlothFS(ro) + roTree = t + errs <- err + }() + + for i := 0; i < cap(errs); i++ { + err := <-errs + if err != nil { + return nil, err + } + } + + if err := createLinks(roTree, rwTree, ro, rw); err != nil { + return nil, err + } + + newInfos := roTree.allFiles() + changed, err := changedFiles(oldInfos, newInfos) + if err != nil { + return nil, fmt.Errorf("changedFiles: %v", err) + } + + for i, p := range changed { + changed[i] = filepath.Join(ro, p) + } + + return changed, nil +}
diff --git a/populate/populate_test.go b/populate/populate_test.go new file mode 100644 index 0000000..141d458 --- /dev/null +++ b/populate/populate_test.go
@@ -0,0 +1,161 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package populate + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "syscall" + "testing" +) + +const attr = "user.gitsha1" +const checksum = "3f75526aa8f01eea5d76cee10722195dc73676de" + +func createFSTree(names []string) (string, error) { + dir, err := ioutil.TempDir("", "") + if err != nil { + return dir, err + } + + for _, f := range names { + p := filepath.Join(dir, f) + if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { + return dir, err + } + if err := ioutil.WriteFile(p, []byte{42}, 0644); err != nil { + return dir, err + } + if err := syscall.Setxattr(p, attr, []byte(checksum), 0); err != nil { + return dir, fmt.Errorf("Setxattr: %v", err) + } + } + return dir, nil +} + +func TestConstruct(t *testing.T) { + dir, err := createFSTree([]string{ + "toplevel", + "build/core/.git/HEAD", + "build/core/subdir/core.h", + "build/core/song/.git/HEAD", + "build/core/song/song.mp3", + "build/core/top", + "build/subfile", + }) + if err != nil { + t.Fatal(err) + } + songT := &repoTree{ + children: map[string]*repoTree{}, + entries: map[string]*fileInfo{ + "song.mp3": &fileInfo{}, + }, + } + coreT := &repoTree{ + children: map[string]*repoTree{"song": songT}, + entries: map[string]*fileInfo{ + "subdir/core.h": &fileInfo{}, + "top": &fileInfo{}, + }, + } + topT := &repoTree{ + children: map[string]*repoTree{ + "build/core": coreT, + }, + entries: map[string]*fileInfo{ + "build/subfile": &fileInfo{}, + "toplevel": &fileInfo{}, + }, + } + + got, err := newRepoTree(dir) + if err != nil { + t.Fatalf("newRepoTree: %v", err) + } + + // Clear unpredictable data. + gotCh := got.allChildren() + + wantCh := topT.allChildren() + for k, v := range wantCh { + if !reflect.DeepEqual(v, gotCh[k]) { + t.Fatalf("subrepo %q: got %#v want %#v", k, gotCh[k], v) + } + } + + if !reflect.DeepEqual(got, topT) { + t.Errorf("got %#v want %#v", got, topT) + } +} + +func TestRepoTreeFromManifest(t *testing.T) { + f, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal("TempFile", err) + } + + _, err = f.Write([]byte(` +<Manifest> + <default revision="master" remote="aosp" dest-branch="" sync-j="4" sync-c="" sync-s=""></default> + <remote alias="" name="aosp" fetch=".." review="https://android-review.googlesource.com/" revision=""></remote> + <project path="build" name="platform/build" groups="pdk,tradefed" revision="55d4a46f6da08b248a467097d56a2762d47d7043" clone-url="https://android.googlesource.com/platform/build"> + <copyfile src="core/root.mk" dest="Makefile"></copyfile> + </project> + <project path="build/blueprint" name="platform/build/blueprint" groups="pdk,tradefed" revision="f0de34718cb9dcb6fbe3bb3afb2a1ef4eae85118" clone-url="https://android.googlesource.com/platform/build/blueprint"> + <linkfile src="root.bp" dest="Android.bp"></linkfile> + </project> + <project path="build/subdir/kati" name="platform/build/subdir/kati" groups="pdk,tradefed" revision="ff2d59e2e082d17ae04f43d409244440a1687856" clone-url="https://android.googlesource.com/platform/build/subdir/kati"></project> +</Manifest>`)) + if err != nil { + t.Fatal("Write", err) + } + + blueprintT := &repoTree{ + children: map[string]*repoTree{}, + entries: map[string]*fileInfo{}, + } + katiT := &repoTree{ + children: map[string]*repoTree{}, + entries: map[string]*fileInfo{}, + } + buildT := &repoTree{ + children: map[string]*repoTree{ + "subdir/kati": katiT, + "blueprint": blueprintT, + }, + entries: map[string]*fileInfo{}, + } + topT := &repoTree{ + children: map[string]*repoTree{ + "build": buildT, + }, + entries: map[string]*fileInfo{ + "Makefile": &fileInfo{}, + "Android.bp": &fileInfo{}, + }, + } + + got, err := repoTreeFromManifest(f.Name()) + if err != nil { + t.Fatalf("repoTreeFromManifest: %v", err) + } + if !reflect.DeepEqual(got, topT) { + t.Errorf("got %#v, want %#v", got, topT) + } +}
diff --git a/populate/repotree.go b/populate/repotree.go new file mode 100644 index 0000000..283350e --- /dev/null +++ b/populate/repotree.go
@@ -0,0 +1,271 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package populate + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + git "github.com/libgit2/git2go" + + "github.com/google/slothfs/gitiles" + "github.com/hanwen/gitfs/manifest" +) + +// fileInfo holds data files contained in the git repository within a +// repoTree node. +type fileInfo struct { + // the SHA1 of the file. This can be nil if getting it was too expensive. + sha1 *git.Oid + + // We can't do chtimes on symlinks. + isLink bool +} + +// repoTree is a nested set of Git repositories. +type repoTree struct { + // repositories under this repository + children map[string]*repoTree + + // files in this repository. + entries map[string]*fileInfo +} + +// findParentRepo recursively finds the deepest child that is a prefix +// to the given path. +func (t *repoTree) findParentRepo(path string) (*repoTree, string) { + for k, ch := range t.children { + if strings.HasPrefix(path, k+"/") { + return ch.findParentRepo(path[len(k+"/"):]) + } + } + return t, path +} + +// write dumps the tree for debugging purposes. +func (t *repoTree) write(w io.Writer, indent string) { + for nm, ch := range t.children { + fmt.Fprintf(w, "%s%s:\n", indent, nm) + ch.write(w, indent+" ") + } +} + +// repoTreeFromManifest creates a repoTree from a manifest XML. +func repoTreeFromManifest(xmlFile string) (*repoTree, error) { + mf, err := manifest.ParseFile(xmlFile) + if err != nil { + return nil, err + } + + var byDepth [][]*manifest.Project + for i, p := range mf.Project { + l := len(strings.Split(p.Path, "/")) + for len(byDepth) <= l { + byDepth = append(byDepth, nil) + } + + byDepth[l] = append(byDepth[l], &mf.Project[i]) + } + + root := makeRepoTree() + treesByPath := map[string]*repoTree{ + "": root, + } + + for _, projs := range byDepth { + for _, p := range projs { + childTree := makeRepoTree() + treesByPath[p.Path] = childTree + + parent, key := root.findParentRepo(p.Path) + parent.children[key] = childTree + } + } + + for _, p := range mf.Project { + for _, c := range p.Copyfile { + root.entries[c.Dest] = &fileInfo{} + } + for _, c := range p.Linkfile { + root.entries[c.Dest] = &fileInfo{} + } + } + return root, nil +} + +// fillFromSlothFS reads tree.json to fill Entries for this repoTree +// node only, and does not recurse. +func (t *repoTree) fillFromSlothFS(dir string) error { + c, err := ioutil.ReadFile(filepath.Join(dir, ".slothfs", "tree.json")) + if err != nil { + return err + } + + var tree gitiles.Tree + if err := json.Unmarshal(c, &tree); err != nil { + return err + } + + for _, e := range tree.Entries { + fi := &fileInfo{} + fi.sha1, err = git.NewOid(e.ID) + if err != nil { + return err + } + + t.entries[e.Name] = fi + + if e.Target != nil { + fi.isLink = true + } + } + + return nil +} + +// repoTreeFromSlothFS reads data from .slothfs to construct a fully +// populated repoTree tree. +func repoTreeFromSlothFS(dir string) (*repoTree, error) { + root, err := repoTreeFromManifest(filepath.Join(dir, ".slothfs", "manifest.xml")) + if err != nil { + return nil, err + } + + chs := root.allChildren() + errs := make(chan error, len(chs)) + for path, ch := range root.allChildren() { + go func(p string, t *repoTree) { + err := t.fillFromSlothFS(p) + errs <- err + }(filepath.Join(dir, path), ch) + } + + for i := 0; i < cap(errs); i++ { + err := <-errs + if err != nil { + return nil, err + } + } + return root, nil +} + +// makeRepoTree returns a repoTree struct with maps initialized. +func makeRepoTree() *repoTree { + return &repoTree{ + children: map[string]*repoTree{}, + entries: map[string]*fileInfo{}, + } +} + +// newRepoTree returns a repoTree constructed from filesystem data. +func newRepoTree(dir string) (*repoTree, error) { + t := makeRepoTree() + if err := t.fill(dir, ""); err != nil { + return nil, err + } + return t, nil +} + +// allChildren returns all the repositories (including the receiver) +// as a map keyed by relative path. +func (t *repoTree) allChildren() map[string]*repoTree { + r := map[string]*repoTree{"": t} + for nm, ch := range t.children { + for sub, subCh := range ch.allChildren() { + r[filepath.Join(nm, sub)] = subCh + } + } + return r +} + +// allFiles returns all the files below this repoTree. +func (t *repoTree) allFiles() map[string]*fileInfo { + r := map[string]*fileInfo{} + for nm, info := range t.entries { + r[nm] = info + } + for nm, ch := range t.children { + for sub, subCh := range ch.allFiles() { + r[filepath.Join(nm, sub)] = subCh + } + } + return r +} + +// returns whether path is the topdirectory of some git repository, +// either in plain git or in slothfs. +func isRepoDir(path string) bool { + if stat, err := os.Stat(filepath.Join(path, ".git")); err == nil && stat.IsDir() { + return true + } else if stat, err := os.Stat(filepath.Join(path, ".slothfs")); err == nil && stat.IsDir() { + return true + } + return false +} + +// construct fills `parent` looking through `dir` subdir of `repoRoot`. +func (parent *repoTree) fill(repoRoot, dir string) error { + entries, err := ioutil.ReadDir(filepath.Join(repoRoot, dir)) + if err != nil { + log.Println(repoRoot, err) + return err + } + + todo := map[string]*repoTree{} + for _, e := range entries { + if e.IsDir() && (e.Name() == ".git" || e.Name() == ".slothfs") { + continue + } + if e.IsDir() && e.Name() == "out" && dir == "" { + // Ignore the build output directory. + continue + } + + subName := filepath.Join(dir, e.Name()) + if e.IsDir() { + if newRoot := filepath.Join(repoRoot, subName); isRepoDir(newRoot) { + ch := makeRepoTree() + parent.children[subName] = ch + todo[newRoot] = ch + } else { + parent.fill(repoRoot, subName) + } + } else { + parent.entries[subName] = &fileInfo{} + } + } + + errs := make(chan error, len(todo)) + for newRoot, ch := range todo { + go func(r string, t *repoTree) { + errs <- t.fill(r, "") + }(newRoot, ch) + } + + for range todo { + err := <-errs + if err != nil { + return err + } + } + + return nil +}