Handle symlinks in populate:
* Accept but ignore Utimens() on directory nodes. This is for symlinks
to include/ directories
* Return added and changed files separately. This is so we don't touch
everything for a sloth-populate call into an empty workspace.
* Cleanup e2e test code, and add a case for a changing symlink.
Change-Id: I743b9887182077bf9c63d660031b92823fbeb1f8
diff --git a/cmd/slothfs-populate/main.go b/cmd/slothfs-populate/main.go
index 5c50d79..c130edf 100644
--- a/cmd/slothfs-populate/main.go
+++ b/cmd/slothfs-populate/main.go
@@ -34,16 +34,24 @@
log.Fatal("too many arguments.")
}
- changed, err := populate.Checkout(*mount, dir)
+ added, changed, err := populate.Checkout(*mount, dir)
if err != nil {
log.Fatalf("populate.Checkout: %v", err)
}
- now := time.Now()
- for _, c := range changed {
- if err := os.Chtimes(c, now, now); err != nil {
- log.Fatalf("Chtimes(%s): %v", c, err)
+ if len(changed) > 0 {
+ now := time.Now()
+ n := 0
+ for _, slice := range [][]string{added, changed} {
+ for _, c := range slice {
+ if err := os.Chtimes(c, now, now); err != nil {
+ log.Fatalf("Chtimes(%s): %v", c, err)
+ }
+ n++
+ }
}
+ log.Printf("touched %d files", n)
+ } else {
+ log.Printf("no files were changed, %d were added; assuming fresh checkout.", len(added))
}
- log.Printf("touched %d files", len(changed))
}
diff --git a/fs/gitilesfs.go b/fs/gitilesfs.go
index de4661c..a43c5fd 100644
--- a/fs/gitilesfs.go
+++ b/fs/gitilesfs.go
@@ -330,6 +330,12 @@
nodefs.Node
}
+// Implement Utimens so we don't create spurious "not implemented"
+// messages when directory targets for symlinks are touched.
+func (n *dirNode) Utimens(file nodefs.File, atime *time.Time, mtime *time.Time, context *fuse.Context) (code fuse.Status) {
+ return fuse.OK
+}
+
func (n *dirNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) {
out.Mode = fuse.S_IFDIR | 0755
t := time.Unix(1, 0)
diff --git a/populate/e2e_test.go b/populate/e2e_test.go
index 47223ff..9f141cf 100644
--- a/populate/e2e_test.go
+++ b/populate/e2e_test.go
@@ -7,12 +7,14 @@
"net"
"os"
"path/filepath"
+ "reflect"
"testing"
"github.com/google/slothfs/cache"
"github.com/google/slothfs/fs"
"github.com/google/slothfs/gitiles"
"github.com/google/slothfs/manifest"
+ "github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/fuse/nodefs"
git "github.com/libgit2/git2go"
@@ -38,58 +40,110 @@
return &i
}
+func newString(s string) *string {
+ return &s
+}
+
func abortListener(l net.Listener) {
- _, err := l.Accept()
- if err == nil {
- log.Panicf("got incoming connection")
+ for {
+ conn, err := l.Accept()
+ if err != nil {
+ break
+ }
+ conn.Close()
}
}
-func TestFUSE(t *testing.T) {
+type fixture struct {
+ dir string
+ cache *cache.Cache
+ fsServer *fuse.Server
+ abortGitiles net.Listener
+}
+
+func (f *fixture) Cleanup() {
+ if f.abortGitiles != nil {
+ f.abortGitiles.Close()
+ }
+ if f.fsServer != nil {
+ if err := f.fsServer.Unmount(); err != nil {
+ return
+ }
+ }
+ os.RemoveAll(f.dir)
+}
+
+func (f *fixture) addWorkspace(name string, mf *manifest.Manifest) error {
+ bytes1, err := mf.MarshalXML()
+ if err != nil {
+ return err
+ }
+
+ dir := f.dir
+
+ if err := ioutil.WriteFile(filepath.Join(dir, name+".xml"), bytes1, 0644); err != nil {
+ return err
+ }
+ if err := os.Symlink(filepath.Join(dir, name+".xml"), filepath.Join(dir, "mnt", "config", name)); err != nil {
+ return err
+ }
+ return nil
+}
+
+func newFixture() (*fixture, error) {
dir, err := ioutil.TempDir("", "")
if err != nil {
- t.Fatal(err)
+ return nil, err
}
- defer os.RemoveAll(dir)
+
+ fix := fixture{dir: dir}
for _, d := range []string{"mnt", "ws", "cache"} {
if err := os.MkdirAll(filepath.Join(dir, d), 0755); err != nil {
- t.Fatal(err)
+ return nil, err
}
}
- cache, err := cache.NewCache(filepath.Join(dir, "cache"), cache.Options{})
+ fix.cache, err = cache.NewCache(filepath.Join(dir, "cache"), cache.Options{})
if err != nil {
- t.Fatal(err)
+ return nil, err
}
// Setup a fake gitiles; make sure we never talk to it.
- l, err := net.Listen("tcp", ":0")
+ fix.abortGitiles, err = net.Listen("tcp", ":0")
if err != nil {
- t.Fatal(err)
+ return nil, err
}
- go abortListener(l)
- defer l.Close()
+ go abortListener(fix.abortGitiles)
- service, err := gitiles.NewService(fmt.Sprintf("http://%s/", l.Addr()), gitiles.Options{})
+ service, err := gitiles.NewService(fmt.Sprintf("http://%s/", fix.abortGitiles.Addr()), gitiles.Options{})
if err != nil {
log.Printf("NewService: %v", err)
}
opts := fs.MultiFSOptions{}
- root := fs.NewMultiFS(service, cache, opts)
+ root := fs.NewMultiFS(service, fix.cache, opts)
fuseOpts := nodefs.NewOptions()
server, _, err := nodefs.MountRoot(filepath.Join(dir, "mnt"), root, fuseOpts)
if err != nil {
- t.Fatal(err)
+ return nil, err
}
go server.Serve()
- defer server.Unmount()
+
+ return &fix, nil
+}
+
+func TestFUSESymlink(t *testing.T) {
+ fixture, err := newFixture()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer fixture.Cleanup()
// We avoid talking to gitiles by inserting entries into the
// cache manually.
- if err := cache.Tree.Add(gitID(ids[0]), &gitiles.Tree{
+ if err := fixture.cache.Tree.Add(gitID(ids[0]), &gitiles.Tree{
ID: ids[0],
Entries: []gitiles.TreeEntry{
{
@@ -100,6 +154,98 @@
Size: newInt(42),
},
{
+ Mode: 0100644,
+ Name: "link",
+ Type: "blob",
+ ID: ids[2],
+ Size: newInt(1),
+ Target: newString("non-existent"),
+ },
+ },
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ // We avoid talking to gitiles by inserting entries into the
+ // cache manually.
+ if err := fixture.cache.Tree.Add(gitID(ids[1]), &gitiles.Tree{
+ ID: ids[1],
+ Entries: []gitiles.TreeEntry{
+ {
+ Mode: 0100644,
+ Name: "a",
+ Type: "blob",
+ ID: ids[1],
+ Size: newInt(42),
+ },
+ {
+ Mode: 0100644,
+ Name: "link",
+ Type: "blob",
+ ID: ids[3],
+ Size: newInt(1),
+ Target: newString("a"),
+ },
+ },
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ for i := 0; i <= 1; i++ {
+ if err := fixture.addWorkspace(fmt.Sprintf("m%d", i), &manifest.Manifest{
+ Project: []manifest.Project{{
+ Name: "platform/project",
+ Path: "p",
+ Revision: ids[i],
+ }}}); err != nil {
+ t.Fatalf("addWorkspace(%d): %v", i, err)
+ }
+ }
+
+ ws := filepath.Join(fixture.dir, "ws")
+ m0 := filepath.Join(fixture.dir, "mnt", "m0")
+ added, changed, err := Checkout(m0, ws)
+ if err != nil {
+ t.Fatalf("Checkout m0: %v", err)
+ }
+ if len(changed) > 0 {
+ t.Errorf("got changed files %v on fresh checkout", changed)
+ }
+ if want := []string{filepath.Join(m0, "p/a"), filepath.Join(m0, "p/link")}; !reflect.DeepEqual(added, want) {
+ t.Errorf("got added %v want %v on fresh checkout", added, want)
+ }
+
+ m1 := filepath.Join(fixture.dir, "mnt", "m1")
+ added, changed, err = Checkout(m1, ws)
+ if len(added) > 0 {
+ t.Errorf("got added files %v on sync", added)
+ }
+ if want := []string{filepath.Join(m1, "p/link")}; !reflect.DeepEqual(changed, want) {
+ t.Errorf("got changed files %v, want %v", changed, want)
+ }
+}
+
+func TestBasic(t *testing.T) {
+ fixture, err := newFixture()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer fixture.Cleanup()
+ dir := fixture.dir
+
+ // We avoid talking to gitiles by inserting entries into the
+ // cache manually.
+ if err := fixture.cache.Tree.Add(gitID(ids[0]), &gitiles.Tree{
+ ID: ids[0],
+ Entries: []gitiles.TreeEntry{
+ {
+ Mode: 0100644,
+ Name: "a",
+ Type: "blob",
+ ID: ids[1],
+ Size: newInt(1),
+ },
+ {
Mode: 0100644,
Name: "b/c",
Type: "blob",
@@ -110,7 +256,7 @@
}); err != nil {
t.Fatal(err)
}
- if err := cache.Tree.Add(gitID(ids[1]), &gitiles.Tree{
+ if err := fixture.cache.Tree.Add(gitID(ids[1]), &gitiles.Tree{
ID: ids[1],
Entries: []gitiles.TreeEntry{
{
@@ -118,7 +264,7 @@
Name: "a",
Type: "blob",
ID: ids[2],
- Size: newInt(42),
+ Size: newInt(1),
},
{
Mode: 0100644,
@@ -132,13 +278,14 @@
Name: "new",
Type: "blob",
ID: ids[3],
- Size: newInt(42),
+ Size: newInt(1),
},
},
}); err != nil {
t.Fatal(err)
}
- if err := cache.Tree.Add(gitID(ids[2]), &gitiles.Tree{
+
+ if err := fixture.cache.Tree.Add(gitID(ids[2]), &gitiles.Tree{
ID: ids[2],
Entries: []gitiles.TreeEntry{
{
@@ -146,21 +293,23 @@
Name: "d",
Type: "blob",
ID: ids[3],
- Size: newInt(42),
+ Size: newInt(1),
},
},
}); err != nil {
t.Fatal(err)
}
- mf1 := manifest.Manifest{
+ if err := fixture.addWorkspace("m1", &manifest.Manifest{
Project: []manifest.Project{{
Name: "platform/project",
Path: "project",
Revision: ids[0],
- }}}
+ }}}); err != nil {
+ t.Fatalf("addWorkspace(m1): %v", err)
+ }
- mf2 := manifest.Manifest{
+ if err := fixture.addWorkspace("m2", &manifest.Manifest{
Project: []manifest.Project{
{
Name: "platform/project",
@@ -171,42 +320,20 @@
Path: "sub",
Revision: ids[2],
}},
- }
-
- bytes1, err := mf1.MarshalXML()
- if err != nil {
- t.Fatal(err)
- }
- if err := ioutil.WriteFile(filepath.Join(dir, "m1.xml"), bytes1, 0644); err != nil {
- t.Fatal(err)
- }
-
- bytes2, err := mf2.MarshalXML()
- if err != nil {
- t.Fatal(err)
- }
- if err := ioutil.WriteFile(filepath.Join(dir, "m2.xml"), bytes2, 0644); err != nil {
- t.Fatal(err)
- }
-
- if err := os.Symlink(filepath.Join(dir, "m1.xml"), filepath.Join(dir, "mnt", "config", "m1")); err != nil {
- t.Fatal(err)
+ }); err != nil {
+ t.Fatalf("addWorkspace(m2): %v", err)
}
testFile := filepath.Join(dir, "mnt", "m1", "project", "b/c")
if fi, err := os.Lstat(testFile); err != nil {
t.Fatalf("Lstat(%s): %v", testFile, err)
} else if fi.Size() != 1 {
- t.Fatalf("%s has size %d", fi.Size())
- }
-
- if err := os.Symlink(filepath.Join(dir, "m2.xml"), filepath.Join(dir, "mnt", "config", "m2")); err != nil {
- t.Fatal(err)
+ t.Fatalf("%s has size %d", testFile, fi.Size())
}
ws := filepath.Join(dir, "ws")
- if _, err := Checkout(filepath.Join(dir, "mnt", "m1"), ws); err != nil {
+ if _, _, err := Checkout(filepath.Join(dir, "mnt", "m1"), ws); err != nil {
t.Fatal("Checkout m1:", err)
}
@@ -220,27 +347,25 @@
// the test setup that no blobs are shared with newly
// appearing files, or they'll be touched for being new files.
- changed, err := Checkout(filepath.Join(dir, "mnt", "m2"), ws)
+ added, changed, err := Checkout(filepath.Join(dir, "mnt", "m2"), ws)
if err != nil {
t.Fatal(err)
}
- for _, f := range []string{"project/a", "project/new"} {
- found := false
- for _, c := range changed {
- if c == filepath.Join(dir, "mnt", "m2", f) {
- found = true
- break
- }
- }
- if !found {
- t.Errorf("file %s was not changed.", f)
- }
+ if want := []string{filepath.Join(dir, "mnt", "m2", "project/a")}; !reflect.DeepEqual(changed, want) {
+ t.Errorf("got changed %v, want %v", changed, want)
+ }
+
+ if want := []string{
+ filepath.Join(dir, "mnt", "m2", "project/new"),
+ filepath.Join(dir, "mnt", "m2", "sub/d"),
+ }; !reflect.DeepEqual(added, want) {
+ t.Errorf("got added %v, want %v", added, want)
}
if dest, err := os.Readlink(filepath.Join(ws, "sub")); err != nil {
t.Fatal(err)
} else if want := filepath.Join(dir, "mnt", "m2", "sub"); dest != want {
- t.Fatalf("got %q, want %q", dest, want)
+ t.Fatalf("Readlink(ws/sub): got %q, want %q", dest, want)
}
}
diff --git a/populate/populate.go b/populate/populate.go
index e8b91a3..37cd82c 100644
--- a/populate/populate.go
+++ b/populate/populate.go
@@ -159,16 +159,11 @@
// Returns the filenames (as relative paths) in newDir that have
// changed relative to the files in oldDir.
-func changedFiles(oldInfos map[string]*fileInfo, newInfos map[string]*fileInfo) ([]string, error) {
- var changed []string
+func changedFiles(oldInfos map[string]*fileInfo, newInfos map[string]*fileInfo) (added, changed []string, err error) {
for path, info := range newInfos {
old, ok := oldInfos[path]
if !ok {
- changed = append(changed, path)
- continue
- }
- if info.isLink {
- // TODO(hanwen): maybe we should we deref the link?
+ added = append(added, path)
continue
}
@@ -182,16 +177,17 @@
}
}
sort.Strings(changed)
- return changed, nil
+ sort.Strings(added)
+ return added, changed, nil
}
// Checkout updates a RW dir with new symlinks to the given RO dir.
// Returns the files that should be touched.
-func Checkout(ro, rw string) ([]string, error) {
+func Checkout(ro, rw string) (added, changed []string, err error) {
ro = filepath.Clean(ro)
wsName, err := clearLinks(filepath.Dir(ro), rw)
if err != nil {
- return nil, err
+ return nil, nil, err
}
oldRoot := filepath.Join(filepath.Dir(ro), wsName)
@@ -227,23 +223,27 @@
for i := 0; i < cap(errs); i++ {
err := <-errs
if err != nil {
- return nil, err
+ return nil, nil, err
}
}
if err := createLinks(roTree, rwTree, ro, rw); err != nil {
- return nil, err
+ return nil, nil, err
}
newInfos := roTree.allFiles()
- changed, err := changedFiles(oldInfos, newInfos)
+ added, changed, err = changedFiles(oldInfos, newInfos)
if err != nil {
- return nil, fmt.Errorf("changedFiles: %v", err)
+ return nil, nil, fmt.Errorf("changedFiles: %v", err)
}
for i, p := range changed {
changed[i] = filepath.Join(ro, p)
}
- return changed, nil
+ for i, p := range added {
+ added[i] = filepath.Join(ro, p)
+ }
+
+ return added, changed, nil
}
