cmd/slothfs-populate/main.go - gitfs - Git at Google

 // Copyright 2016 Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package main

 import (
 	"bytes"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"syscall"
 	"time"

 	git "github.com/libgit2/git2go"
 )

 type fileInfo struct {
 	isRegular bool
 	size      int64
 	inode     uint64
 }

 type repoTree struct {
 	// repositories under this repository
 	children map[string]*repoTree

 	// files in this repository.
 	entries map[string]*fileInfo
 }

 func makeRepoTree() *repoTree {
 	return &repoTree{
 		children: map[string]*repoTree{},
 		entries:  map[string]*fileInfo{},
 	}
 }

 func newRepoTree(dir string) (*repoTree, error) {
 	t := makeRepoTree()
 	if err := t.fill(dir, ""); err != nil {
 		return nil, err
 	}
 	return t, nil
 }

 // allChildren returns all the repositories (including the receiver)
 // as a map keyed by relative path.
 func (t *repoTree) allChildren() map[string]*repoTree {
 	r := map[string]*repoTree{"": t}
 	for nm, ch := range t.children {
 		for sub, subCh := range ch.allChildren() {
 			r[filepath.Join(nm, sub)] = subCh
 		}
 	}
 	return r
 }

 // allFiles returns all the files below this repoTree.
 func (t *repoTree) allFiles() map[string]*fileInfo {
 	r := map[string]*fileInfo{}
 	for nm, info := range t.entries {
 		r[nm] = info
 	}
 	for nm, ch := range t.children {
 		for sub, subCh := range ch.allFiles() {
 			r[filepath.Join(nm, sub)] = subCh
 		}
 	}
 	return r
 }

 func isRepoDir(path string) bool {
 	if stat, err := os.Stat(filepath.Join(path, ".git")); err == nil && stat.IsDir() {
 		return true
 	} else if stat, err := os.Stat(filepath.Join(path, ".gitid")); err == nil && !stat.IsDir() {
 		return true
 	}
 	return false
 }

 // construct fills `parent` looking through `dir` subdir of `repoRoot`.
 func (parent *repoTree) fill(repoRoot, dir string) error {
 	entries, err := ioutil.ReadDir(filepath.Join(repoRoot, dir))
 	if err != nil {
 		return err
 	}

 	todo := map[string]*repoTree{}
 	for _, e := range entries {
 		if (e.IsDir() && e.Name() == ".git") || (!e.IsDir() && e.Name() == ".gitid") {
 			continue
 		}
 		if e.IsDir() && e.Name() == "out" && dir == "" {
 			// Ignore the build output directory.
 			continue
 		}

 		subName := filepath.Join(dir, e.Name())
 		if e.IsDir() {
 			if newRoot := filepath.Join(repoRoot, subName); isRepoDir(newRoot) {
 				ch := makeRepoTree()
 				parent.children[subName] = ch
 				todo[newRoot] = ch
 			} else {
 				parent.fill(repoRoot, subName)
 			}
 		} else {
 			parent.entries[subName] = &fileInfo{
 				isRegular: e.Mode()&os.ModeType == 0,
 				size:      e.Size(),
 				inode:     e.Sys().(*syscall.Stat_t).Ino,
 			}
 		}
 	}

 	errs := make(chan error, len(todo))
 	for newRoot, ch := range todo {
 		go func(r string, t *repoTree) {
 			errs <- t.fill(r, "")
 		}(newRoot, ch)
 	}

 	for range todo {
 		err := <-errs
 		if err != nil {
 			return err
 		}
 	}

 	return nil
 }

 // symlinkRepo creates symlinks for all the files in `child`.
 func symlinkRepo(name string, child *repoTree, roRoot, rwRoot string) error {
 	fi, err := os.Stat(filepath.Join(rwRoot, name))
 	if err == nil && fi.IsDir() {
 		return nil
 	}

 	for e := range child.entries {
 		dest := filepath.Join(rwRoot, name, e)

 		if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
 			return err
 		}
 		if err := os.Symlink(filepath.Join(roRoot, name, e), dest); err != nil {
 			return err
 		}
 	}
 	return nil
 }

 // createTreeLinks tries to short-cut symlinks for whole trees by
 // symlinking to the root of a repository in the RO tree.
 func createTreeLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
 	allRW := rw.allChildren()

 outer:
 	for nm, ch := range ro.children {
 		foundCheckout := false
 		foundRecurse := false
 		for k := range allRW {
 			if k == "" {
 				continue
 			}
 			if nm == k {
 				foundRecurse = true
 				break
 			}
 			rel, err := filepath.Rel(nm, k)
 			if err != nil {
 				return err
 			}

 			if strings.HasPrefix(rel, "..") {
 				continue
 			}

 			// we have a checkout below "nm".
 			foundCheckout = true
 			break
 		}

 		switch {
 		case foundRecurse:
 			if err := createTreeLinks(ch, rw.children[nm], filepath.Join(roRoot, nm), filepath.Join(rwRoot, nm)); err != nil {
 				return err
 			}
 			continue outer
 		case !foundCheckout:
 			dest := filepath.Join(rwRoot, nm)
 			if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
 				return err
 			}
 			if err := os.Symlink(filepath.Join(roRoot, nm), dest); err != nil {
 				return err
 			}
 		}
 	}
 	return nil
 }

 // createLinks will populate a RW tree with symlinks to the RO tree.
 func createLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
 	if err := createTreeLinks(ro, rw, roRoot, rwRoot); err != nil {
 		return err
 	}

 	rwc := rw.allChildren()
 	for nm, ch := range ro.allChildren() {
 		if _, ok := rwc[nm]; !ok {
 			if err := symlinkRepo(nm, ch, roRoot, rwRoot); err != nil {
 				return err
 			}
 		}
 	}
 	return nil
 }

 // clearLinks removes all symlinks to the RO tree. It returns the workspace name that was linked before.
 func clearLinks(mount, dir string) (string, error) {
 	mount = filepath.Clean(mount)

 	var prefix string
 	var dirs []string
 	if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error {
 		if fi.Mode()&os.ModeSymlink != 0 {
 			target, err := os.Readlink(n)
 			if err != nil {
 				return err
 			}
 			if strings.HasPrefix(target, mount) {
 				prefix = target
 				if err := os.Remove(n); err != nil {
 					return err
 				}
 			}
 		}
 		if fi.IsDir() {
 			dirs = append(dirs, n)
 		}
 		return nil
 	}); err != nil {
 		return "", err
 	}

 	// Reverse the ordering, so we get the deepest subdirs first.
 	sort.Strings(dirs)
 	for i := range dirs {
 		d := dirs[len(dirs)-1-i]
 		// Ignore error: dir may still contain entries.
 		os.Remove(d)
 	}

 	prefix = strings.TrimPrefix(prefix, mount+"/")
 	if i := strings.Index(prefix, "/"); i != -1 {
 		prefix = prefix[:i]
 	}
 	return prefix, nil
 }

 const attrName = "user.gitsha1"

 func getSHA1(fn string) (*git.Oid, error) {
 	var data [40]byte
 	sz, err := syscall.Getxattr(fn, attrName, data[:])
 	if err != nil {
 		return nil, fmt.Errorf("Getxattr(%s, %s): %v", fn, attrName, err)
 	}

 	oid, err := git.NewOid(string(data[:sz]))
 	if err != nil {
 		return nil, err
 	}
 	return oid, nil
 }

 // Returns the filenames (as relative paths) in newDir that have
 // changed relative to the files in oldDir.
 func changedFiles(oldDir string, oldInfos map[string]*fileInfo,
 	newDir string, newInfos map[string]*fileInfo) ([]string, error) {
 	var changed []string
 	for path, info := range newInfos {
 		if path == "manifest.xml" {
 			continue
 		}

 		if !info.isRegular {
 			// TODO(hanwen): this is incorrect. If a file
 			// changes from a blob to a symlink, we should
 			// deref the symlink and check if the blob has
 			// changed.
 			continue
 		}

 		old, ok := oldInfos[path]
 		if !ok {
 			changed = append(changed, path)
 			continue
 		}

 		if old.inode == info.inode {
 			continue
 		}

 		if old.size != info.size {
 			changed = append(changed, path)
 			continue
 		}

 		oldSHA1, err := getSHA1(filepath.Join(oldDir, path))
 		if err != nil {
 			return nil, err
 		}
 		newSHA1, err := getSHA1(filepath.Join(newDir, path))
 		if err != nil {
 			return nil, err
 		}

 		if bytes.Compare(oldSHA1[:], newSHA1[:]) != 0 {
 			changed = append(changed, path)
 		}
 	}

 	sort.Strings(changed)
 	return changed, nil
 }

 // populateCheckout updates a RW dir with new symlinks to the given RO dir.
 func populateCheckout(ro, rw string) error {
 	ro = filepath.Clean(ro)
 	wsName, err := clearLinks(filepath.Dir(ro), rw)
 	if err != nil {
 		return err
 	}
 	oldRoot := filepath.Join(filepath.Dir(ro), wsName)

 	// Do the file system traversals in parallel.
 	errs := make(chan error, 3)
 	var rwTree, roTree *repoTree
 	var oldInfos map[string]*fileInfo

 	if wsName != "" {
 		go func() {
 			t, err := newRepoTree(oldRoot)
 			oldInfos = t.allFiles()
 			errs <- err
 		}()
 	} else {
 		oldInfos = map[string]*fileInfo{}
 		errs <- nil
 	}

 	go func() {
 		t, err := newRepoTree(rw)
 		rwTree = t
 		errs <- err
 	}()
 	go func() {
 		t, err := newRepoTree(ro)
 		roTree = t
 		errs <- err
 	}()

 	for i := 0; i < cap(errs); i++ {
 		err := <-errs
 		if err != nil {
 			return err
 		}
 	}

 	if err := createLinks(roTree, rwTree, ro, rw); err != nil {
 		return err
 	}

 	changed, err := changedFiles(oldRoot, oldInfos, ro, roTree.allFiles())
 	if err != nil {
 		return fmt.Errorf("changedFiles: %v", err)
 	}

 	for i, p := range changed {
 		changed[i] = filepath.Join(ro, p)
 	}

 	if err := seqTouch(changed, time.Now()); err != nil {
 		return err
 	}

 	return nil
 }

 func seqTouch(fs []string, t time.Time) error {
 	for _, f := range fs {
 		if err := os.Chtimes(f, t, t); err != nil {
 			return err
 		}
 	}

 	return nil
 }

 func main() {
 	mount := flag.String("ro", "", "path to slothfs-multifs mount.")
 	flag.Parse()

 	dir := "."
 	if len(flag.Args()) == 1 {
 		dir = flag.Arg(0)
 	} else if len(flag.Args()) > 1 {
 		log.Fatal("too many arguments.")
 	}

 	if err := populateCheckout(*mount, dir); err != nil {
 		log.Fatalf("populateCheckout: %v", err)
 	}
 }
	// Copyright 2016 Google Inc. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package main

	import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	git "github.com/libgit2/git2go"
	)

	type fileInfo struct {
	isRegular bool
	size int64
	inode uint64
	}

	type repoTree struct {
	// repositories under this repository
	children map[string]*repoTree

	// files in this repository.
	entries map[string]*fileInfo
	}

	func makeRepoTree() *repoTree {
	return &repoTree{
	children: map[string]*repoTree{},
	entries: map[string]*fileInfo{},
	}
	}

	func newRepoTree(dir string) (*repoTree, error) {
	t := makeRepoTree()
	if err := t.fill(dir, ""); err != nil {
	return nil, err
	}
	return t, nil
	}

	// allChildren returns all the repositories (including the receiver)
	// as a map keyed by relative path.
	func (t repoTree) allChildren() map[string]repoTree {
	r := map[string]*repoTree{"": t}
	for nm, ch := range t.children {
	for sub, subCh := range ch.allChildren() {
	r[filepath.Join(nm, sub)] = subCh
	}
	}
	return r
	}

	// allFiles returns all the files below this repoTree.
	func (t repoTree) allFiles() map[string]fileInfo {
	r := map[string]*fileInfo{}
	for nm, info := range t.entries {
	r[nm] = info
	}
	for nm, ch := range t.children {
	for sub, subCh := range ch.allFiles() {
	r[filepath.Join(nm, sub)] = subCh
	}
	}
	return r
	}

	func isRepoDir(path string) bool {
	if stat, err := os.Stat(filepath.Join(path, ".git")); err == nil && stat.IsDir() {
	return true
	} else if stat, err := os.Stat(filepath.Join(path, ".gitid")); err == nil && !stat.IsDir() {
	return true
	}
	return false
	}

	// construct fills `parent` looking through `dir` subdir of `repoRoot`.
	func (parent *repoTree) fill(repoRoot, dir string) error {
	entries, err := ioutil.ReadDir(filepath.Join(repoRoot, dir))
	if err != nil {
	return err
	}

	todo := map[string]*repoTree{}
	for _, e := range entries {
	if (e.IsDir() && e.Name() == ".git") \|\| (!e.IsDir() && e.Name() == ".gitid") {
	continue
	}
	if e.IsDir() && e.Name() == "out" && dir == "" {
	// Ignore the build output directory.
	continue
	}

	subName := filepath.Join(dir, e.Name())
	if e.IsDir() {
	if newRoot := filepath.Join(repoRoot, subName); isRepoDir(newRoot) {
	ch := makeRepoTree()
	parent.children[subName] = ch
	todo[newRoot] = ch
	} else {
	parent.fill(repoRoot, subName)
	}
	} else {
	parent.entries[subName] = &fileInfo{
	isRegular: e.Mode()&os.ModeType == 0,
	size: e.Size(),
	inode: e.Sys().(*syscall.Stat_t).Ino,
	}
	}
	}

	errs := make(chan error, len(todo))
	for newRoot, ch := range todo {
	go func(r string, t *repoTree) {
	errs <- t.fill(r, "")
	}(newRoot, ch)
	}

	for range todo {
	err := <-errs
	if err != nil {
	return err
	}
	}

	return nil
	}

	// symlinkRepo creates symlinks for all the files in `child`.
	func symlinkRepo(name string, child *repoTree, roRoot, rwRoot string) error {
	fi, err := os.Stat(filepath.Join(rwRoot, name))
	if err == nil && fi.IsDir() {
	return nil
	}

	for e := range child.entries {
	dest := filepath.Join(rwRoot, name, e)

	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
	return err
	}
	if err := os.Symlink(filepath.Join(roRoot, name, e), dest); err != nil {
	return err
	}
	}
	return nil
	}

	// createTreeLinks tries to short-cut symlinks for whole trees by
	// symlinking to the root of a repository in the RO tree.
	func createTreeLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
	allRW := rw.allChildren()

	outer:
	for nm, ch := range ro.children {
	foundCheckout := false
	foundRecurse := false
	for k := range allRW {
	if k == "" {
	continue
	}
	if nm == k {
	foundRecurse = true
	break
	}
	rel, err := filepath.Rel(nm, k)
	if err != nil {
	return err
	}

	if strings.HasPrefix(rel, "..") {
	continue
	}

	// we have a checkout below "nm".
	foundCheckout = true
	break
	}

	switch {
	case foundRecurse:
	if err := createTreeLinks(ch, rw.children[nm], filepath.Join(roRoot, nm), filepath.Join(rwRoot, nm)); err != nil {
	return err
	}
	continue outer
	case !foundCheckout:
	dest := filepath.Join(rwRoot, nm)
	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
	return err
	}
	if err := os.Symlink(filepath.Join(roRoot, nm), dest); err != nil {
	return err
	}
	}
	}
	return nil
	}

	// createLinks will populate a RW tree with symlinks to the RO tree.
	func createLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
	if err := createTreeLinks(ro, rw, roRoot, rwRoot); err != nil {
	return err
	}

	rwc := rw.allChildren()
	for nm, ch := range ro.allChildren() {
	if _, ok := rwc[nm]; !ok {
	if err := symlinkRepo(nm, ch, roRoot, rwRoot); err != nil {
	return err
	}
	}
	}
	return nil
	}

	// clearLinks removes all symlinks to the RO tree. It returns the workspace name that was linked before.
	func clearLinks(mount, dir string) (string, error) {
	mount = filepath.Clean(mount)

	var prefix string
	var dirs []string
	if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error {
	if fi.Mode()&os.ModeSymlink != 0 {
	target, err := os.Readlink(n)
	if err != nil {
	return err
	}
	if strings.HasPrefix(target, mount) {
	prefix = target
	if err := os.Remove(n); err != nil {
	return err
	}
	}
	}
	if fi.IsDir() {
	dirs = append(dirs, n)
	}
	return nil
	}); err != nil {
	return "", err
	}

	// Reverse the ordering, so we get the deepest subdirs first.
	sort.Strings(dirs)
	for i := range dirs {
	d := dirs[len(dirs)-1-i]
	// Ignore error: dir may still contain entries.
	os.Remove(d)
	}

	prefix = strings.TrimPrefix(prefix, mount+"/")
	if i := strings.Index(prefix, "/"); i != -1 {
	prefix = prefix[:i]
	}
	return prefix, nil
	}

	const attrName = "user.gitsha1"

	func getSHA1(fn string) (*git.Oid, error) {
	var data [40]byte
	sz, err := syscall.Getxattr(fn, attrName, data[:])
	if err != nil {
	return nil, fmt.Errorf("Getxattr(%s, %s): %v", fn, attrName, err)
	}

	oid, err := git.NewOid(string(data[:sz]))
	if err != nil {
	return nil, err
	}
	return oid, nil
	}

	// Returns the filenames (as relative paths) in newDir that have
	// changed relative to the files in oldDir.
	func changedFiles(oldDir string, oldInfos map[string]*fileInfo,
	newDir string, newInfos map[string]*fileInfo) ([]string, error) {
	var changed []string
	for path, info := range newInfos {
	if path == "manifest.xml" {
	continue
	}

	if !info.isRegular {
	// TODO(hanwen): this is incorrect. If a file
	// changes from a blob to a symlink, we should
	// deref the symlink and check if the blob has
	// changed.
	continue
	}

	old, ok := oldInfos[path]
	if !ok {
	changed = append(changed, path)
	continue
	}

	if old.inode == info.inode {
	continue
	}

	if old.size != info.size {
	changed = append(changed, path)
	continue
	}

	oldSHA1, err := getSHA1(filepath.Join(oldDir, path))
	if err != nil {
	return nil, err
	}
	newSHA1, err := getSHA1(filepath.Join(newDir, path))
	if err != nil {
	return nil, err
	}

	if bytes.Compare(oldSHA1[:], newSHA1[:]) != 0 {
	changed = append(changed, path)
	}
	}

	sort.Strings(changed)
	return changed, nil
	}

	// populateCheckout updates a RW dir with new symlinks to the given RO dir.
	func populateCheckout(ro, rw string) error {
	ro = filepath.Clean(ro)
	wsName, err := clearLinks(filepath.Dir(ro), rw)
	if err != nil {
	return err
	}
	oldRoot := filepath.Join(filepath.Dir(ro), wsName)

	// Do the file system traversals in parallel.
	errs := make(chan error, 3)
	var rwTree, roTree *repoTree
	var oldInfos map[string]*fileInfo

	if wsName != "" {
	go func() {
	t, err := newRepoTree(oldRoot)
	oldInfos = t.allFiles()
	errs <- err
	}()
	} else {
	oldInfos = map[string]*fileInfo{}
	errs <- nil
	}

	go func() {
	t, err := newRepoTree(rw)
	rwTree = t
	errs <- err
	}()
	go func() {
	t, err := newRepoTree(ro)
	roTree = t
	errs <- err
	}()

	for i := 0; i < cap(errs); i++ {
	err := <-errs
	if err != nil {
	return err
	}
	}

	if err := createLinks(roTree, rwTree, ro, rw); err != nil {
	return err
	}

	changed, err := changedFiles(oldRoot, oldInfos, ro, roTree.allFiles())
	if err != nil {
	return fmt.Errorf("changedFiles: %v", err)
	}

	for i, p := range changed {
	changed[i] = filepath.Join(ro, p)
	}

	if err := seqTouch(changed, time.Now()); err != nil {
	return err
	}

	return nil
	}

	func seqTouch(fs []string, t time.Time) error {
	for _, f := range fs {
	if err := os.Chtimes(f, t, t); err != nil {
	return err
	}
	}

	return nil
	}

	func main() {
	mount := flag.String("ro", "", "path to slothfs-multifs mount.")
	flag.Parse()

	dir := "."
	if len(flag.Args()) == 1 {
	dir = flag.Arg(0)
	} else if len(flag.Args()) > 1 {
	log.Fatal("too many arguments.")
	}

	if err := populateCheckout(*mount, dir); err != nil {
	log.Fatalf("populateCheckout: %v", err)
	}
	}