cmd/zoekt-mirror-bitbucket-server: delete stale repos
This change adds a method to delete stale repos for the BitBucket Server mirror.
Closes #91.
Change-Id: Iff3e643e3aec16c19fb51b89aab77212063815b6
diff --git a/cmd/zoekt-indexserver/config.go b/cmd/zoekt-indexserver/config.go
index 0a73bef..d976904 100644
--- a/cmd/zoekt-indexserver/config.go
+++ b/cmd/zoekt-indexserver/config.go
@@ -195,7 +195,7 @@
cmd.Args = append(cmd.Args, c.CGitURL)
} else if c.BitBucketServerURL != "" {
cmd = exec.Command("zoekt-mirror-bitbucket-server",
- "-dest", repoDir, "-url", c.BitBucketServerURL)
+ "-dest", repoDir, "-url", c.BitBucketServerURL, "-delete")
if c.BitBucketServerProject != "" {
cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject)
}
diff --git a/cmd/zoekt-mirror-bitbucket-server/main.go b/cmd/zoekt-mirror-bitbucket-server/main.go
index 790d075..19270ec 100644
--- a/cmd/zoekt-mirror-bitbucket-server/main.go
+++ b/cmd/zoekt-mirror-bitbucket-server/main.go
@@ -1,5 +1,3 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -41,6 +39,7 @@
serverUrl := flag.String("url", "", "BitBucket Server url")
credentialsFile := flag.String("credentials", ".bitbucket-credentials", "file holding BitBucket Server credentials")
project := flag.String("project", "", "project to mirror")
+ deleteRepos := flag.Bool("delete", false, "delete missing repos")
namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
projectType := flag.String("type", "", "only clone repos whose type matches the given string. "+
@@ -137,6 +136,36 @@
if err := cloneRepos(destDir, rootURL.Host, repos, password); err != nil {
log.Fatalf("cloneRepos: %v", err)
}
+
+ if *deleteRepos {
+ if err := deleteStaleRepos(*dest, filter, repos); err != nil {
+ log.Fatalf("deleteStaleRepos: %v", err)
+ }
+ }
+}
+
+func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []bitbucketv1.Repository) error {
+ var baseURL string
+ if len(repos) > 0 {
+ baseURL = repos[0].Links.Self[0].Href
+ } else {
+ return nil
+ }
+ u, err := url.Parse(baseURL)
+ if err != nil {
+ return err
+ }
+ u.Path = ""
+
+ names := map[string]struct{}{}
+ for _, r := range repos {
+ names[filepath.Join(u.Host, r.Project.Key, r.Slug+".git")] = struct{}{}
+ }
+
+ if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
+ log.Fatalf("deleteRepos: %v", err)
+ }
+ return nil
}
func IsValidProjectType(projectType string) bool {
diff --git a/cmd/zoekt-mirror-github/main.go b/cmd/zoekt-mirror-github/main.go
index c1848ca..7a2078b 100644
--- a/cmd/zoekt-mirror-github/main.go
+++ b/cmd/zoekt-mirror-github/main.go
@@ -170,39 +170,17 @@
}
u.Path = user
- paths, err := gitindex.ListRepos(destDir, u)
- if err != nil {
- return err
- }
-
- names := map[string]bool{}
+ names := map[string]struct{}{}
for _, r := range repos {
u, err := url.Parse(*r.HTMLURL)
if err != nil {
return err
}
- names[filepath.Join(u.Host, u.Path+".git")] = true
+ names[filepath.Join(u.Host, u.Path+".git")] = struct{}{}
}
- var toDelete []string
- for _, p := range paths {
- if filter.Include(filepath.Base(p)) && !names[p] {
- toDelete = append(toDelete, p)
- }
- }
-
- if len(toDelete) > 0 {
- log.Printf("deleting repos %v", toDelete)
- }
-
- var errs []string
- for _, d := range toDelete {
- if err := os.RemoveAll(filepath.Join(destDir, d)); err != nil {
- errs = append(errs, err.Error())
- }
- }
- if len(errs) > 0 {
- return fmt.Errorf("errors: %v", errs)
+ if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
+ log.Fatalf("deleteRepos: %v", err)
}
return nil
}
diff --git a/cmd/zoekt-mirror-gitlab/main.go b/cmd/zoekt-mirror-gitlab/main.go
index 8e156de..7f1fa40 100644
--- a/cmd/zoekt-mirror-gitlab/main.go
+++ b/cmd/zoekt-mirror-gitlab/main.go
@@ -146,40 +146,18 @@
return err
}
- paths, err := gitindex.ListRepos(destDir, u)
- if err != nil {
- return err
- }
-
- names := map[string]bool{}
+ names := map[string]struct{}{}
for _, p := range projects {
u, err := url.Parse(p.HTTPURLToRepo)
if err != nil {
return err
}
- names[filepath.Join(u.Host, u.Path)] = true
+ names[filepath.Join(u.Host, u.Path)] = struct{}{}
}
- var toDelete []string
- for _, p := range paths {
- if filter.Include(p) && !names[p] {
- toDelete = append(toDelete, p)
- }
- }
-
- if len(toDelete) > 0 {
- log.Printf("deleting repos %v", toDelete)
- }
-
- var errs []string
- for _, d := range toDelete {
- if err := os.RemoveAll(filepath.Join(destDir, d)); err != nil {
- errs = append(errs, err.Error())
- }
- }
- if len(errs) > 0 {
- return fmt.Errorf("errors: %v", errs)
+ if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
+ log.Fatalf("deleteRepos: %v", err)
}
return nil
}
diff --git a/gitindex/delete.go b/gitindex/delete.go
new file mode 100644
index 0000000..f53e1af
--- /dev/null
+++ b/gitindex/delete.go
@@ -0,0 +1,41 @@
+package gitindex
+
+import (
+ "fmt"
+ "log"
+ "net/url"
+ "os"
+ "path/filepath"
+)
+
+// DeleteRepos deletes stale repos under a specific path in disk. The `names`
+// argument stores names of repos retrieved from the git hosting site
+// and is used along with the `filter` argument to decide on repo deletion.
+func DeleteRepos(baseDir string, urlPrefix *url.URL, names map[string]struct{}, filter *Filter) error {
+ paths, err := ListRepos(baseDir, urlPrefix)
+ if err != nil {
+ return err
+ }
+ var toDelete []string
+ for _, p := range paths {
+ _, exists := names[p]
+ if filter.Include(filepath.Base(p)) && !exists {
+ toDelete = append(toDelete, p)
+ }
+ }
+
+ if len(toDelete) > 0 {
+ log.Printf("deleting repos %v", toDelete)
+ }
+
+ var errs []string
+ for _, d := range toDelete {
+ if err := os.RemoveAll(filepath.Join(baseDir, d)); err != nil {
+ errs = append(errs, err.Error())
+ }
+ }
+ if len(errs) > 0 {
+ return fmt.Errorf("errors: %v", errs)
+ }
+ return nil
+}
diff --git a/gitindex/delete_test.go b/gitindex/delete_test.go
new file mode 100644
index 0000000..571a25d
--- /dev/null
+++ b/gitindex/delete_test.go
@@ -0,0 +1,83 @@
+package gitindex
+
+import (
+ "io/ioutil"
+ "net/url"
+ "path/filepath"
+ "reflect"
+ "testing"
+)
+
+func TestDeleteRepos(t *testing.T) {
+ dir, err := ioutil.TempDir("", "")
+ if err != nil {
+ t.Fatalf("TempDir: %v", err)
+ }
+
+ if err := createSubmoduleRepo(dir); err != nil {
+ t.Error("createSubmoduleRepo", err)
+ }
+
+ reposBefore, err := FindGitRepos(dir)
+ if err != nil {
+ t.Error("FindGitRepos", err)
+ }
+
+ gotBefore := map[string]struct{}{}
+ for _, r := range reposBefore {
+ p, err := filepath.Rel(dir, r)
+ if err != nil {
+ t.Fatalf("Relative: %v", err)
+ }
+
+ gotBefore[p] = struct{}{}
+ }
+
+ wantBefore := map[string]struct{}{
+ "gerrit.googlesource.com/bdir.git": {},
+ "gerrit.googlesource.com/sub/bdir.git": {},
+ "adir/.git": {},
+ "bdir/.git": {},
+ "gerrit.googlesource.com/adir.git": {},
+ }
+
+ if !reflect.DeepEqual(gotBefore, wantBefore) {
+ t.Fatalf("got %v want %v", gotBefore, wantBefore)
+ }
+
+ aURL, _ := url.Parse("http://gerrit.googlesource.com")
+ aURL.Path = "sub"
+ names := map[string]struct{}{
+ "bdir/.git": {},
+ "gerrit.googlesource.com/adir.git": {},
+ }
+ filter, _ := NewFilter("", "")
+
+ err = DeleteRepos(dir, aURL, names, filter)
+ if err != nil {
+ t.Fatalf("DeleteRepos: %T", err)
+ }
+ reposAfter, err := FindGitRepos(dir)
+ if err != nil {
+ t.Error("FindGitRepos", err)
+ }
+
+ gotAfter := map[string]struct{}{}
+ for _, r := range reposAfter {
+ p, err := filepath.Rel(dir, r)
+ if err != nil {
+ t.Fatalf("Relative: %v", err)
+ }
+
+ gotAfter[p] = struct{}{}
+ }
+ wantAfter := map[string]struct{}{
+ "gerrit.googlesource.com/bdir.git": {},
+ "adir/.git": {},
+ "bdir/.git": {},
+ "gerrit.googlesource.com/adir.git": {}}
+
+ if !reflect.DeepEqual(gotAfter, wantAfter) {
+ t.Errorf("got %v want %v", gotAfter, wantAfter)
+ }
+}
