cmd/zoekt-mirror-gitlab/main.go

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary fetches all repos for a user from gitlab.
//
// It is recommended to use a gitlab personal access token:
// https://docs.gitlab.com/ce/user/profile/personal_access_tokens.html. This
// token should be stored in a file and the --token option should be used.
// In addition, the token should be present in the ~/.netrc of the user running
// the mirror command. For example, the ~/.netrc may look like:
//
//   machine gitlab.com
//   login oauth
//   password <personal access token>
//
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/google/zoekt/gitindex"
	gitlab "github.com/xanzy/go-gitlab"
)

func main() {
	dest := flag.String("dest", "", "destination directory")
	gitlabURL := flag.String("url", "https://gitlab.com/api/v4/", "Gitlab URL. If not set https://gitlab.com/api/v4/ will be used")
	token := flag.String("token",
		filepath.Join(os.Getenv("HOME"), ".gitlab-token"),
		"file holding API token.")
	isMember := flag.Bool("membership", false, "only mirror repos this user is a member of ")
	isPublic := flag.Bool("public", false, "only mirror public repos")
	deleteRepos := flag.Bool("delete", false, "delete missing repos")
	namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
	flag.Parse()

	if *dest == "" {
		log.Fatal("must set --dest")
	}

	var host string
	rootURL, err := url.Parse(*gitlabURL)
	if err != nil {
		log.Fatal(err)
	}
	host = rootURL.Host

	destDir := filepath.Join(*dest, host)
	if err := os.MkdirAll(destDir, 0755); err != nil {
		log.Fatal(err)
	}

	content, err := ioutil.ReadFile(*token)
	if err != nil {
		log.Fatal(err)
	}
	apiToken := strings.TrimSpace(string(content))

	client := gitlab.NewClient(nil, apiToken)
	client.SetBaseURL(*gitlabURL)

	opt := &gitlab.ListProjectsOptions{
		ListOptions: gitlab.ListOptions{
			PerPage: 10,
			Page:    1,
		},
		Membership: isMember,
	}
	if *isPublic {
		opt.Visibility = gitlab.Visibility(gitlab.PublicVisibility)
	}

	var gitlabProjects []*gitlab.Project
	for {
		projects, resp, err := client.Projects.ListProjects(opt)

		if err != nil {
			log.Fatal(err)
		}

		for _, project := range projects {

			// Skip projects without a default branch - these should be projects
			// where the repository isn't enabled
			if project.DefaultBranch == "" {
				continue
			}

			gitlabProjects = append(gitlabProjects, project)
		}

		if resp.CurrentPage >= resp.TotalPages {
			break
		}

		opt.Page = resp.NextPage
	}

	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
	if err != nil {
		log.Fatal(err)
	}

	{
		trimmed := gitlabProjects[:0]
		for _, p := range gitlabProjects {
			if filter.Include(p.NameWithNamespace) {
				trimmed = append(trimmed, p)
			}
		}
		gitlabProjects = trimmed
	}

	fetchProjects(destDir, apiToken, gitlabProjects)

	if *deleteRepos {
		if err := deleteStaleProjects(*dest, filter, gitlabProjects); err != nil {
			log.Fatalf("deleteStaleProjects: %v", err)
		}
	}
}

func deleteStaleProjects(destDir string, filter *gitindex.Filter, projects []*gitlab.Project) error {

	u, err := url.Parse(projects[0].HTTPURLToRepo)
	u.Path = ""
	if err != nil {
		return err
	}

	names := map[string]struct{}{}
	for _, p := range projects {
		u, err := url.Parse(p.HTTPURLToRepo)
		if err != nil {
			return err
		}

		names[filepath.Join(u.Host, u.Path)] = struct{}{}
	}

	if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
		log.Fatalf("deleteRepos: %v", err)
	}
	return nil
}

func fetchProjects(destDir, token string, projects []*gitlab.Project) {

	for _, p := range projects {
		u, err := url.Parse(p.HTTPURLToRepo)
		if err != nil {
			log.Printf("Unable to parse project URL: %v", err)
			continue
		}
		config := map[string]string{
			"zoekt.web-url-type": "gitlab",
			"zoekt.web-url":      p.WebURL,
			"zoekt.name":         filepath.Join(u.Hostname(), p.PathWithNamespace),

			"zoekt.gitlab-stars": strconv.Itoa(p.StarCount),
			"zoekt.gitlab-forks": strconv.Itoa(p.ForksCount),
		}

		cloneURL := p.HTTPURLToRepo
		dest, err := gitindex.CloneRepo(destDir, p.PathWithNamespace, cloneURL, config)
		if err != nil {
			log.Printf("cloneRepos: %v", err)
			continue
		}
		if dest != "" {
			fmt.Println(dest)
		}
	}
}