cmd/zoekt-mirror-bitbucket-server: delete stale repos This change adds a method to delete stale repos for the BitBucket Server mirror. Closes #91. Change-Id: Iff3e643e3aec16c19fb51b89aab77212063815b6
diff --git a/cmd/zoekt-indexserver/config.go b/cmd/zoekt-indexserver/config.go index 0a73bef..d976904 100644 --- a/cmd/zoekt-indexserver/config.go +++ b/cmd/zoekt-indexserver/config.go
@@ -195,7 +195,7 @@ cmd.Args = append(cmd.Args, c.CGitURL) } else if c.BitBucketServerURL != "" { cmd = exec.Command("zoekt-mirror-bitbucket-server", - "-dest", repoDir, "-url", c.BitBucketServerURL) + "-dest", repoDir, "-url", c.BitBucketServerURL, "-delete") if c.BitBucketServerProject != "" { cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject) }
diff --git a/cmd/zoekt-mirror-bitbucket-server/main.go b/cmd/zoekt-mirror-bitbucket-server/main.go index 790d075..19270ec 100644 --- a/cmd/zoekt-mirror-bitbucket-server/main.go +++ b/cmd/zoekt-mirror-bitbucket-server/main.go
@@ -1,5 +1,3 @@ -// Copyright 2016 Google Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -41,6 +39,7 @@ serverUrl := flag.String("url", "", "BitBucket Server url") credentialsFile := flag.String("credentials", ".bitbucket-credentials", "file holding BitBucket Server credentials") project := flag.String("project", "", "project to mirror") + deleteRepos := flag.Bool("delete", false, "delete missing repos") namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.") excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") projectType := flag.String("type", "", "only clone repos whose type matches the given string. "+ @@ -137,6 +136,36 @@ if err := cloneRepos(destDir, rootURL.Host, repos, password); err != nil { log.Fatalf("cloneRepos: %v", err) } + + if *deleteRepos { + if err := deleteStaleRepos(*dest, filter, repos); err != nil { + log.Fatalf("deleteStaleRepos: %v", err) + } + } +} + +func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []bitbucketv1.Repository) error { + var baseURL string + if len(repos) > 0 { + baseURL = repos[0].Links.Self[0].Href + } else { + return nil + } + u, err := url.Parse(baseURL) + if err != nil { + return err + } + u.Path = "" + + names := map[string]struct{}{} + for _, r := range repos { + names[filepath.Join(u.Host, r.Project.Key, r.Slug+".git")] = struct{}{} + } + + if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil { + log.Fatalf("deleteRepos: %v", err) + } + return nil } func IsValidProjectType(projectType string) bool {
diff --git a/cmd/zoekt-mirror-github/main.go b/cmd/zoekt-mirror-github/main.go index c1848ca..7a2078b 100644 --- a/cmd/zoekt-mirror-github/main.go +++ b/cmd/zoekt-mirror-github/main.go
@@ -170,39 +170,17 @@ } u.Path = user - paths, err := gitindex.ListRepos(destDir, u) - if err != nil { - return err - } - - names := map[string]bool{} + names := map[string]struct{}{} for _, r := range repos { u, err := url.Parse(*r.HTMLURL) if err != nil { return err } - names[filepath.Join(u.Host, u.Path+".git")] = true + names[filepath.Join(u.Host, u.Path+".git")] = struct{}{} } - var toDelete []string - for _, p := range paths { - if filter.Include(filepath.Base(p)) && !names[p] { - toDelete = append(toDelete, p) - } - } - - if len(toDelete) > 0 { - log.Printf("deleting repos %v", toDelete) - } - - var errs []string - for _, d := range toDelete { - if err := os.RemoveAll(filepath.Join(destDir, d)); err != nil { - errs = append(errs, err.Error()) - } - } - if len(errs) > 0 { - return fmt.Errorf("errors: %v", errs) + if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil { + log.Fatalf("deleteRepos: %v", err) } return nil }
diff --git a/cmd/zoekt-mirror-gitlab/main.go b/cmd/zoekt-mirror-gitlab/main.go index 8e156de..7f1fa40 100644 --- a/cmd/zoekt-mirror-gitlab/main.go +++ b/cmd/zoekt-mirror-gitlab/main.go
@@ -146,40 +146,18 @@ return err } - paths, err := gitindex.ListRepos(destDir, u) - if err != nil { - return err - } - - names := map[string]bool{} + names := map[string]struct{}{} for _, p := range projects { u, err := url.Parse(p.HTTPURLToRepo) if err != nil { return err } - names[filepath.Join(u.Host, u.Path)] = true + names[filepath.Join(u.Host, u.Path)] = struct{}{} } - var toDelete []string - for _, p := range paths { - if filter.Include(p) && !names[p] { - toDelete = append(toDelete, p) - } - } - - if len(toDelete) > 0 { - log.Printf("deleting repos %v", toDelete) - } - - var errs []string - for _, d := range toDelete { - if err := os.RemoveAll(filepath.Join(destDir, d)); err != nil { - errs = append(errs, err.Error()) - } - } - if len(errs) > 0 { - return fmt.Errorf("errors: %v", errs) + if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil { + log.Fatalf("deleteRepos: %v", err) } return nil }
diff --git a/gitindex/delete.go b/gitindex/delete.go new file mode 100644 index 0000000..f53e1af --- /dev/null +++ b/gitindex/delete.go
@@ -0,0 +1,41 @@ +package gitindex + +import ( + "fmt" + "log" + "net/url" + "os" + "path/filepath" +) + +// DeleteRepos deletes stale repos under a specific path in disk. The `names` +// argument stores names of repos retrieved from the git hosting site +// and is used along with the `filter` argument to decide on repo deletion. +func DeleteRepos(baseDir string, urlPrefix *url.URL, names map[string]struct{}, filter *Filter) error { + paths, err := ListRepos(baseDir, urlPrefix) + if err != nil { + return err + } + var toDelete []string + for _, p := range paths { + _, exists := names[p] + if filter.Include(filepath.Base(p)) && !exists { + toDelete = append(toDelete, p) + } + } + + if len(toDelete) > 0 { + log.Printf("deleting repos %v", toDelete) + } + + var errs []string + for _, d := range toDelete { + if err := os.RemoveAll(filepath.Join(baseDir, d)); err != nil { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("errors: %v", errs) + } + return nil +}
diff --git a/gitindex/delete_test.go b/gitindex/delete_test.go new file mode 100644 index 0000000..571a25d --- /dev/null +++ b/gitindex/delete_test.go
@@ -0,0 +1,83 @@ +package gitindex + +import ( + "io/ioutil" + "net/url" + "path/filepath" + "reflect" + "testing" +) + +func TestDeleteRepos(t *testing.T) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("TempDir: %v", err) + } + + if err := createSubmoduleRepo(dir); err != nil { + t.Error("createSubmoduleRepo", err) + } + + reposBefore, err := FindGitRepos(dir) + if err != nil { + t.Error("FindGitRepos", err) + } + + gotBefore := map[string]struct{}{} + for _, r := range reposBefore { + p, err := filepath.Rel(dir, r) + if err != nil { + t.Fatalf("Relative: %v", err) + } + + gotBefore[p] = struct{}{} + } + + wantBefore := map[string]struct{}{ + "gerrit.googlesource.com/bdir.git": {}, + "gerrit.googlesource.com/sub/bdir.git": {}, + "adir/.git": {}, + "bdir/.git": {}, + "gerrit.googlesource.com/adir.git": {}, + } + + if !reflect.DeepEqual(gotBefore, wantBefore) { + t.Fatalf("got %v want %v", gotBefore, wantBefore) + } + + aURL, _ := url.Parse("http://gerrit.googlesource.com") + aURL.Path = "sub" + names := map[string]struct{}{ + "bdir/.git": {}, + "gerrit.googlesource.com/adir.git": {}, + } + filter, _ := NewFilter("", "") + + err = DeleteRepos(dir, aURL, names, filter) + if err != nil { + t.Fatalf("DeleteRepos: %T", err) + } + reposAfter, err := FindGitRepos(dir) + if err != nil { + t.Error("FindGitRepos", err) + } + + gotAfter := map[string]struct{}{} + for _, r := range reposAfter { + p, err := filepath.Rel(dir, r) + if err != nil { + t.Fatalf("Relative: %v", err) + } + + gotAfter[p] = struct{}{} + } + wantAfter := map[string]struct{}{ + "gerrit.googlesource.com/bdir.git": {}, + "adir/.git": {}, + "bdir/.git": {}, + "gerrit.googlesource.com/adir.git": {}} + + if !reflect.DeepEqual(gotAfter, wantAfter) { + t.Errorf("got %v want %v", gotAfter, wantAfter) + } +}