Share nodes with equal IDs between checkouts. Change-Id: I30284cd7a58f31fa8f2aa566a53fcec4d99bda25
diff --git a/fs/gitilesfs.go b/fs/gitilesfs.go index f9d5d1e..443f1f7 100644 --- a/fs/gitilesfs.go +++ b/fs/gitilesfs.go
@@ -35,6 +35,8 @@ type gitilesRoot struct { nodefs.Node + nodeCache *nodeCache + cache *cache.Cache service *gitiles.RepoService tree *gitiles.Tree @@ -244,13 +246,14 @@ // NewGitilesRoot returns the root node for a file system. func NewGitilesRoot(c *cache.Cache, tree *gitiles.Tree, service *gitiles.RepoService, options GitilesOptions) nodefs.Node { r := &gitilesRoot{ - Node: nodefs.NewDefaultNode(), - service: service, - cache: c, - shaMap: map[git.Oid]string{}, - tree: tree, - opts: options, - lazyRepo: cache.NewLazyRepo(options.CloneURL, c), + Node: nodefs.NewDefaultNode(), + service: service, + nodeCache: newNodeCache(), + cache: c, + shaMap: map[git.Oid]string{}, + tree: tree, + opts: options, + lazyRepo: cache.NewLazyRepo(options.CloneURL, c), } return r @@ -306,27 +309,36 @@ } } - n := &gitilesNode{ - Node: nodefs.NewDefaultNode(), - id: *id, - mode: uint32(e.Mode), - clone: clone, - root: r, - // Ninja uses mtime == 0 as "doesn't exist" - // flag, (see ninja/files/src/graph.h:66), so - // use a nonzero timestamp here. - mtime: time.Unix(1, 0), - } - if e.Size != nil { - n.size = int64(*e.Size) + xbit := e.Mode&0111 != 0 + n := r.nodeCache.get(id, xbit) + if n == nil { + n = &gitilesNode{ + Node: nodefs.NewDefaultNode(), + id: *id, + mode: uint32(e.Mode), + clone: clone, + root: r, + // Ninja uses mtime == 0 as "doesn't exist" + // flag, (see ninja/files/src/graph.h:66), so + // use a nonzero timestamp here. + mtime: time.Unix(1, 0), + } + if e.Size != nil { + n.size = int64(*e.Size) + } + + if e.Target != nil { + n.linkTarget = []byte(*e.Target) + n.size = int64(len(n.linkTarget)) + } + + r.shaMap[*id] = p + parent.NewChild(base, false, n) + r.nodeCache.add(n) + } else { + parent.AddChild(base, n.Inode()) } - if e.Target != nil { - n.linkTarget = []byte(*e.Target) - n.size = int64(len(n.linkTarget)) - } - r.shaMap[*id] = p - parent.NewChild(base, false, n) } r.Inode().NewChild(".gitid",
diff --git a/fs/gitilesfs_test.go b/fs/gitilesfs_test.go index 49dc122..915420e 100644 --- a/fs/gitilesfs_test.go +++ b/fs/gitilesfs_test.go
@@ -40,8 +40,10 @@ func init() { enc := map[string]string{ - "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORS?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`, - "/platform/build/kati/+/ce34badf691d36e8048b63f89d1a86ee5fa4325c/testcase/addprefix.mk": "dGVzdDoKCWVjaG8gJChhZGRwcmVmaXggc3JjLyxmb28gYmFyKQo=", + "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORS?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`, + "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORSx?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`, + "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORS2?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`, + "/platform/build/kati/+/ce34badf691d36e8048b63f89d1a86ee5fa4325c/testcase/addprefix.mk": "dGVzdDoKCWVjaG8gJChhZGRwcmVmaXggc3JjLyxmb28gYmFyKQo=", } for k, v := range enc { c := make([]byte, base64.StdEncoding.DecodedLen(len(v))) @@ -106,6 +108,20 @@ { "mode": 33188, "type": "blob", + "id": "787d767f94fd634ed29cd69ec9f93bab2b25f5d4", + "name": "AUTHORS2", + "size": 373 + }, + { + "mode": 33261, + "type": "blob", + "id": "787d767f94fd634ed29cd69ec9f93bab2b25f5d4", + "name": "AUTHORSx", + "size": 373 + }, + { + "mode": 33188, + "type": "blob", "id": "91c29720b08211898308eb2b6bde8bd3208c6dcd", "name": "Android.bp", "size": 1935 @@ -163,6 +179,45 @@ return listener, err } +func TestGitilesFSSharedNodes(t *testing.T) { + fix, err := newTestFixture() + if err != nil { + t.Fatal("newTestFixture", err) + } + defer fix.cleanup() + + repoService := fix.service.NewRepoService("platform/build/kati") + treeResp, err := repoService.GetTree("ce34badf691d36e8048b63f89d1a86ee5fa4325c", "", true) + if err != nil { + t.Fatal("Tree:", err) + } + + options := GitilesOptions{} + + fs := NewGitilesRoot(fix.cache, treeResp, repoService, options) + if err := fix.mount(fs); err != nil { + t.Fatal("mount", err) + } + + ch1 := fs.Inode().GetChild("AUTHORS") + if ch1 == nil { + t.Fatalf("node for AUTHORS not found") + } + + ch2 := fs.Inode().GetChild("AUTHORS2") + if ch2 == nil { + t.Fatalf("node for AUTHORS2 not found") + } + + if ch1 != ch2 { + t.Error("equal blobs did not share inodes.") + } + ch3 := fs.Inode().GetChild("AUTHORSx") + if ch1 == ch3 { + t.Error("blob with different modes shared inode.") + } +} + func TestGitilesFSTreeID(t *testing.T) { fix, err := newTestFixture() if err != nil {
diff --git a/fs/manifestfs.go b/fs/manifestfs.go index 7eae4b0..ec5d973 100644 --- a/fs/manifestfs.go +++ b/fs/manifestfs.go
@@ -33,7 +33,8 @@ service *gitiles.Service - cache *cache.Cache + cache *cache.Cache + nodeCache *nodeCache // trees is Path => Tree map. trees map[string]*gitiles.Tree @@ -58,6 +59,7 @@ } root := &manifestFSRoot{ Node: nodefs.NewDefaultNode(), + nodeCache: newNodeCache(), cache: cache, service: service, options: opts, @@ -147,6 +149,7 @@ } subRoot := NewGitilesRoot(fs.cache, fs.trees[p], repoService, opts) + subRoot.(*gitilesRoot).nodeCache = fs.nodeCache parent.NewChild(base, true, subRoot) if err := subRoot.(*gitilesRoot).onMount(fsConn); err != nil { return fmt.Errorf("onMount(%s): %v", p, err)
diff --git a/fs/multifs.go b/fs/multifs.go index 97babe9..444b49a 100644 --- a/fs/multifs.go +++ b/fs/multifs.go
@@ -28,10 +28,11 @@ type multiManifestFSRoot struct { nodefs.Node - cache *cache.Cache - fsConn *nodefs.FileSystemConnector - options MultiFSOptions - gitiles *gitiles.Service + nodeCache *nodeCache + cache *cache.Cache + fsConn *nodefs.FileSystemConnector + options MultiFSOptions + gitiles *gitiles.Service } func (r *multiManifestFSRoot) OnMount(fsConn *nodefs.FileSystemConnector) { @@ -46,10 +47,11 @@ func NewMultiFS(service *gitiles.Service, c *cache.Cache, options MultiFSOptions) *multiManifestFSRoot { r := &multiManifestFSRoot{ - Node: nodefs.NewDefaultNode(), - cache: c, - options: options, - gitiles: service, + Node: nodefs.NewDefaultNode(), + nodeCache: newNodeCache(), + cache: c, + options: options, + gitiles: service, } return r } @@ -122,6 +124,7 @@ log.Printf("NewManifestFS(%s): %v", string(content), err) return nil, fuse.EIO } + fs.(*manifestFSRoot).nodeCache = c.root.nodeCache ch := c.root.Inode().NewChild(name, true, fs) if ch == nil { @@ -146,7 +149,3 @@ return config, fuse.OK } - -// TODO(hanwen): implement configNode.Unlink - -// TODO(hanwen): make sure content nodes are shared between workspaces.
diff --git a/fs/nodecache.go b/fs/nodecache.go new file mode 100644 index 0000000..2b94e85 --- /dev/null +++ b/fs/nodecache.go
@@ -0,0 +1,60 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "sync" + + "github.com/libgit2/git2go" +) + +type nodeCacheKey struct { + ID git.Oid + xbit bool +} + +// The nodeCache keeps a map of ID to FS node. It is safe for +// concurrent use from multiple goroutines. The cache allows us to +// reuse out the same node for multiple files, effectively +// hard-linking the file. This is done for two reasons: first, each +// blob takes up kernel FS cache memory only once, even if it may be +// used in multiple checkouts. Second, moving data from the FUSE +// process into the kernel is relatively expensive. Thus, we can +// amortize the cost of the read over multiple checkouts. +type nodeCache struct { + mu sync.RWMutex + nodeMap map[nodeCacheKey]*gitilesNode +} + +func newNodeCache() *nodeCache { + return &nodeCache{ + nodeMap: make(map[nodeCacheKey]*gitilesNode), + } +} + +func (c *nodeCache) get(id *git.Oid, xbit bool) *gitilesNode { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.nodeMap[nodeCacheKey{*id, xbit}] +} + +func (c *nodeCache) add(n *gitilesNode) { + xbit := n.mode&0111 != 0 + c.mu.Lock() + defer c.mu.Unlock() + + c.nodeMap[nodeCacheKey{n.id, xbit}] = n +}