Share nodes with equal IDs between checkouts.
Change-Id: I30284cd7a58f31fa8f2aa566a53fcec4d99bda25
diff --git a/fs/gitilesfs.go b/fs/gitilesfs.go
index f9d5d1e..443f1f7 100644
--- a/fs/gitilesfs.go
+++ b/fs/gitilesfs.go
@@ -35,6 +35,8 @@
type gitilesRoot struct {
nodefs.Node
+ nodeCache *nodeCache
+
cache *cache.Cache
service *gitiles.RepoService
tree *gitiles.Tree
@@ -244,13 +246,14 @@
// NewGitilesRoot returns the root node for a file system.
func NewGitilesRoot(c *cache.Cache, tree *gitiles.Tree, service *gitiles.RepoService, options GitilesOptions) nodefs.Node {
r := &gitilesRoot{
- Node: nodefs.NewDefaultNode(),
- service: service,
- cache: c,
- shaMap: map[git.Oid]string{},
- tree: tree,
- opts: options,
- lazyRepo: cache.NewLazyRepo(options.CloneURL, c),
+ Node: nodefs.NewDefaultNode(),
+ service: service,
+ nodeCache: newNodeCache(),
+ cache: c,
+ shaMap: map[git.Oid]string{},
+ tree: tree,
+ opts: options,
+ lazyRepo: cache.NewLazyRepo(options.CloneURL, c),
}
return r
@@ -306,27 +309,36 @@
}
}
- n := &gitilesNode{
- Node: nodefs.NewDefaultNode(),
- id: *id,
- mode: uint32(e.Mode),
- clone: clone,
- root: r,
- // Ninja uses mtime == 0 as "doesn't exist"
- // flag, (see ninja/files/src/graph.h:66), so
- // use a nonzero timestamp here.
- mtime: time.Unix(1, 0),
- }
- if e.Size != nil {
- n.size = int64(*e.Size)
+ xbit := e.Mode&0111 != 0
+ n := r.nodeCache.get(id, xbit)
+ if n == nil {
+ n = &gitilesNode{
+ Node: nodefs.NewDefaultNode(),
+ id: *id,
+ mode: uint32(e.Mode),
+ clone: clone,
+ root: r,
+ // Ninja uses mtime == 0 as "doesn't exist"
+ // flag, (see ninja/files/src/graph.h:66), so
+ // use a nonzero timestamp here.
+ mtime: time.Unix(1, 0),
+ }
+ if e.Size != nil {
+ n.size = int64(*e.Size)
+ }
+
+ if e.Target != nil {
+ n.linkTarget = []byte(*e.Target)
+ n.size = int64(len(n.linkTarget))
+ }
+
+ r.shaMap[*id] = p
+ parent.NewChild(base, false, n)
+ r.nodeCache.add(n)
+ } else {
+ parent.AddChild(base, n.Inode())
}
- if e.Target != nil {
- n.linkTarget = []byte(*e.Target)
- n.size = int64(len(n.linkTarget))
- }
- r.shaMap[*id] = p
- parent.NewChild(base, false, n)
}
r.Inode().NewChild(".gitid",
diff --git a/fs/gitilesfs_test.go b/fs/gitilesfs_test.go
index 49dc122..915420e 100644
--- a/fs/gitilesfs_test.go
+++ b/fs/gitilesfs_test.go
@@ -40,8 +40,10 @@
func init() {
enc := map[string]string{
- "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORS?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`,
- "/platform/build/kati/+/ce34badf691d36e8048b63f89d1a86ee5fa4325c/testcase/addprefix.mk": "dGVzdDoKCWVjaG8gJChhZGRwcmVmaXggc3JjLyxmb28gYmFyKQo=",
+ "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORS?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`,
+ "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORSx?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`,
+ "/platform/build/kati/+show/ce34badf691d36e8048b63f89d1a86ee5fa4325c/AUTHORS2?format=TEXT": `IyBUaGlzIGlzIHRoZSBvZmZpY2lhbCBsaXN0IG9mIGdsb2cgYXV0aG9ycyBmb3IgY29weXJpZ2h0IHB1cnBvc2VzLgojIFRoaXMgZmlsZSBpcyBkaXN0aW5jdCBmcm9tIHRoZSBDT05UUklCVVRPUlMgZmlsZXMuCiMgU2VlIHRoZSBsYXR0ZXIgZm9yIGFuIGV4cGxhbmF0aW9uLgojCiMgTmFtZXMgc2hvdWxkIGJlIGFkZGVkIHRvIHRoaXMgZmlsZSBhczoKIwlOYW1lIG9yIE9yZ2FuaXphdGlvbiA8ZW1haWwgYWRkcmVzcz4KIyBUaGUgZW1haWwgYWRkcmVzcyBpcyBub3QgcmVxdWlyZWQgZm9yIG9yZ2FuaXphdGlvbnMuCiMKIyBQbGVhc2Uga2VlcCB0aGUgbGlzdCBzb3J0ZWQuCgpLb3VoZWkgU3V0b3UgPGtvdUBjb3ptaXhuZy5vcmc+Ckdvb2dsZSBJbmMuCg==`,
+ "/platform/build/kati/+/ce34badf691d36e8048b63f89d1a86ee5fa4325c/testcase/addprefix.mk": "dGVzdDoKCWVjaG8gJChhZGRwcmVmaXggc3JjLyxmb28gYmFyKQo=",
}
for k, v := range enc {
c := make([]byte, base64.StdEncoding.DecodedLen(len(v)))
@@ -106,6 +108,20 @@
{
"mode": 33188,
"type": "blob",
+ "id": "787d767f94fd634ed29cd69ec9f93bab2b25f5d4",
+ "name": "AUTHORS2",
+ "size": 373
+ },
+ {
+ "mode": 33261,
+ "type": "blob",
+ "id": "787d767f94fd634ed29cd69ec9f93bab2b25f5d4",
+ "name": "AUTHORSx",
+ "size": 373
+ },
+ {
+ "mode": 33188,
+ "type": "blob",
"id": "91c29720b08211898308eb2b6bde8bd3208c6dcd",
"name": "Android.bp",
"size": 1935
@@ -163,6 +179,45 @@
return listener, err
}
+func TestGitilesFSSharedNodes(t *testing.T) {
+ fix, err := newTestFixture()
+ if err != nil {
+ t.Fatal("newTestFixture", err)
+ }
+ defer fix.cleanup()
+
+ repoService := fix.service.NewRepoService("platform/build/kati")
+ treeResp, err := repoService.GetTree("ce34badf691d36e8048b63f89d1a86ee5fa4325c", "", true)
+ if err != nil {
+ t.Fatal("Tree:", err)
+ }
+
+ options := GitilesOptions{}
+
+ fs := NewGitilesRoot(fix.cache, treeResp, repoService, options)
+ if err := fix.mount(fs); err != nil {
+ t.Fatal("mount", err)
+ }
+
+ ch1 := fs.Inode().GetChild("AUTHORS")
+ if ch1 == nil {
+ t.Fatalf("node for AUTHORS not found")
+ }
+
+ ch2 := fs.Inode().GetChild("AUTHORS2")
+ if ch2 == nil {
+ t.Fatalf("node for AUTHORS2 not found")
+ }
+
+ if ch1 != ch2 {
+ t.Error("equal blobs did not share inodes.")
+ }
+ ch3 := fs.Inode().GetChild("AUTHORSx")
+ if ch1 == ch3 {
+ t.Error("blob with different modes shared inode.")
+ }
+}
+
func TestGitilesFSTreeID(t *testing.T) {
fix, err := newTestFixture()
if err != nil {
diff --git a/fs/manifestfs.go b/fs/manifestfs.go
index 7eae4b0..ec5d973 100644
--- a/fs/manifestfs.go
+++ b/fs/manifestfs.go
@@ -33,7 +33,8 @@
service *gitiles.Service
- cache *cache.Cache
+ cache *cache.Cache
+ nodeCache *nodeCache
// trees is Path => Tree map.
trees map[string]*gitiles.Tree
@@ -58,6 +59,7 @@
}
root := &manifestFSRoot{
Node: nodefs.NewDefaultNode(),
+ nodeCache: newNodeCache(),
cache: cache,
service: service,
options: opts,
@@ -147,6 +149,7 @@
}
subRoot := NewGitilesRoot(fs.cache, fs.trees[p], repoService, opts)
+ subRoot.(*gitilesRoot).nodeCache = fs.nodeCache
parent.NewChild(base, true, subRoot)
if err := subRoot.(*gitilesRoot).onMount(fsConn); err != nil {
return fmt.Errorf("onMount(%s): %v", p, err)
diff --git a/fs/multifs.go b/fs/multifs.go
index 97babe9..444b49a 100644
--- a/fs/multifs.go
+++ b/fs/multifs.go
@@ -28,10 +28,11 @@
type multiManifestFSRoot struct {
nodefs.Node
- cache *cache.Cache
- fsConn *nodefs.FileSystemConnector
- options MultiFSOptions
- gitiles *gitiles.Service
+ nodeCache *nodeCache
+ cache *cache.Cache
+ fsConn *nodefs.FileSystemConnector
+ options MultiFSOptions
+ gitiles *gitiles.Service
}
func (r *multiManifestFSRoot) OnMount(fsConn *nodefs.FileSystemConnector) {
@@ -46,10 +47,11 @@
func NewMultiFS(service *gitiles.Service, c *cache.Cache, options MultiFSOptions) *multiManifestFSRoot {
r := &multiManifestFSRoot{
- Node: nodefs.NewDefaultNode(),
- cache: c,
- options: options,
- gitiles: service,
+ Node: nodefs.NewDefaultNode(),
+ nodeCache: newNodeCache(),
+ cache: c,
+ options: options,
+ gitiles: service,
}
return r
}
@@ -122,6 +124,7 @@
log.Printf("NewManifestFS(%s): %v", string(content), err)
return nil, fuse.EIO
}
+ fs.(*manifestFSRoot).nodeCache = c.root.nodeCache
ch := c.root.Inode().NewChild(name, true, fs)
if ch == nil {
@@ -146,7 +149,3 @@
return config, fuse.OK
}
-
-// TODO(hanwen): implement configNode.Unlink
-
-// TODO(hanwen): make sure content nodes are shared between workspaces.
diff --git a/fs/nodecache.go b/fs/nodecache.go
new file mode 100644
index 0000000..2b94e85
--- /dev/null
+++ b/fs/nodecache.go
@@ -0,0 +1,60 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fs
+
+import (
+ "sync"
+
+ "github.com/libgit2/git2go"
+)
+
+type nodeCacheKey struct {
+ ID git.Oid
+ xbit bool
+}
+
+// The nodeCache keeps a map of ID to FS node. It is safe for
+// concurrent use from multiple goroutines. The cache allows us to
+// reuse out the same node for multiple files, effectively
+// hard-linking the file. This is done for two reasons: first, each
+// blob takes up kernel FS cache memory only once, even if it may be
+// used in multiple checkouts. Second, moving data from the FUSE
+// process into the kernel is relatively expensive. Thus, we can
+// amortize the cost of the read over multiple checkouts.
+type nodeCache struct {
+ mu sync.RWMutex
+ nodeMap map[nodeCacheKey]*gitilesNode
+}
+
+func newNodeCache() *nodeCache {
+ return &nodeCache{
+ nodeMap: make(map[nodeCacheKey]*gitilesNode),
+ }
+}
+
+func (c *nodeCache) get(id *git.Oid, xbit bool) *gitilesNode {
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+
+ return c.nodeMap[nodeCacheKey{*id, xbit}]
+}
+
+func (c *nodeCache) add(n *gitilesNode) {
+ xbit := n.mode&0111 != 0
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ c.nodeMap[nodeCacheKey{n.id, xbit}] = n
+}
