diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 7cd8594..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,6 +0,0 @@
-*~
-cmd/zoekt-index/zoekt-index
-cmd/zoekt-webserver/zoekt-webserver
-cmd/zoekt-mirror-github/zoekt-mirror-github
-cmd/zoekt-server/zoekt-server
-cmd/zoekt-git-index/zoekt-git-index
diff --git a/CONTRIBUTING b/CONTRIBUTING
deleted file mode 100644
index 61b75c4..0000000
--- a/CONTRIBUTING
+++ /dev/null
@@ -1,4 +0,0 @@
-**NOTICE:**
-[github.com/sourcegraph/zoekt](https://github.com/sourcegraph/zoekt) is the
-active main repository for Zoekt development. Please contribute pull requests
-to that repository.
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index 261eeb9..0000000
--- a/LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/README.md b/README.md
index 6050602..3ab4d09 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,3 @@
-
-    "Zoekt, en gij zult spinazie eten" - Jan Eertink
-
-    ("seek, and ye shall eat spinach" - My primary school teacher)
-
 This is a fast text search engine, intended for use with source
 code. (Pronunciation: roughly as you would pronounce "zooked" in English)
 
diff --git a/all.bash b/all.bash
deleted file mode 100755
index 4b5ae45..0000000
--- a/all.bash
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/sh
-set -eux
-go test github.com/google/zoekt/...
-go install github.com/google/zoekt/cmd/...
diff --git a/api.go b/api.go
deleted file mode 100644
index 2bd8a88..0000000
--- a/api.go
+++ /dev/null
@@ -1,312 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/google/zoekt/query"
-)
-
-// FileMatch contains all the matches within a file.
-type FileMatch struct {
-	// Ranking; the higher, the better.
-	Score float64 // TODO - hide this field?
-
-	// For debugging. Needs DebugScore set, but public so tests in
-	// other packages can print some diagnostics.
-	Debug string
-
-	FileName string
-
-	// Repository is the globally unique name of the repo of the
-	// match
-	Repository  string
-	Branches    []string
-	LineMatches []LineMatch
-
-	// Only set if requested
-	Content []byte
-
-	// Checksum of the content.
-	Checksum []byte
-
-	// Detected language of the result.
-	Language string
-
-	// SubRepositoryName is the globally unique name of the repo,
-	// if it came from a subrepository
-	SubRepositoryName string
-
-	// SubRepositoryPath holds the prefix where the subrepository
-	// was mounted.
-	SubRepositoryPath string
-
-	// Commit SHA1 (hex) of the (sub)repo holding the file.
-	Version string
-}
-
-// LineMatch holds the matches within a single line in a file.
-type LineMatch struct {
-	// The line in which a match was found.
-	Line       []byte
-	LineStart  int
-	LineEnd    int
-	LineNumber int
-
-	// If set, this was a match on the filename.
-	FileName bool
-
-	// The higher the better. Only ranks the quality of the match
-	// within the file, does not take rank of file into account
-	Score         float64
-	LineFragments []LineFragmentMatch
-}
-
-// LineFragmentMatch a segment of matching text within a line.
-type LineFragmentMatch struct {
-	// Offset within the line, in bytes.
-	LineOffset int
-
-	// Offset from file start, in bytes.
-	Offset uint32
-
-	// Number bytes that match.
-	MatchLength int
-}
-
-// Stats contains interesting numbers on the search
-type Stats struct {
-	// Amount of I/O for reading contents.
-	ContentBytesLoaded int64
-
-	// Amount of I/O for reading from index.
-	IndexBytesLoaded int64
-
-	// Number of search shards that had a crash.
-	Crashes int
-
-	// Wall clock time for this search
-	Duration time.Duration
-
-	// Number of files containing a match.
-	FileCount int
-
-	// Number of files in shards that we considered.
-	ShardFilesConsidered int
-
-	// Files that we evaluated. Equivalent to files for which all
-	// atom matches (including negations) evaluated to true.
-	FilesConsidered int
-
-	// Files for which we loaded file content to verify substring matches
-	FilesLoaded int
-
-	// Candidate files whose contents weren't examined because we
-	// gathered enough matches.
-	FilesSkipped int
-
-	// Shards that we did not process because a query was canceled.
-	ShardsSkipped int
-
-	// Number of non-overlapping matches
-	MatchCount int
-
-	// Number of candidate matches as a result of searching ngrams.
-	NgramMatches int
-
-	// Wall clock time for queued search.
-	Wait time.Duration
-
-	// Number of times regexp was called on files that we evaluated.
-	RegexpsConsidered int
-}
-
-func (s *Stats) Add(o Stats) {
-	s.ContentBytesLoaded += o.ContentBytesLoaded
-	s.IndexBytesLoaded += o.IndexBytesLoaded
-	s.Crashes += o.Crashes
-	s.FileCount += o.FileCount
-	s.FilesConsidered += o.FilesConsidered
-	s.FilesLoaded += o.FilesLoaded
-	s.FilesSkipped += o.FilesSkipped
-	s.MatchCount += o.MatchCount
-	s.NgramMatches += o.NgramMatches
-	s.ShardFilesConsidered += o.ShardFilesConsidered
-	s.ShardsSkipped += o.ShardsSkipped
-}
-
-// SearchResult contains search matches and extra data
-type SearchResult struct {
-	Stats
-	Files []FileMatch
-
-	// RepoURLs holds a repo => template string map.
-	RepoURLs map[string]string
-
-	// FragmentNames holds a repo => template string map, for
-	// the line number fragment.
-	LineFragments map[string]string
-}
-
-// RepositoryBranch describes an indexed branch, which is a name
-// combined with a version.
-type RepositoryBranch struct {
-	Name    string
-	Version string
-}
-
-// Repository holds repository metadata.
-type Repository struct {
-	// The repository name
-	Name string
-
-	// The repository URL.
-	URL string
-
-	// The physical source where this repo came from, eg. full
-	// path to the zip filename or git repository directory. This
-	// will not be exposed in the UI, but can be used to detect
-	// orphaned index shards.
-	Source string
-
-	// The branches indexed in this repo.
-	Branches []RepositoryBranch
-
-	// Nil if this is not the super project.
-	SubRepoMap map[string]*Repository
-
-	// URL template to link to the commit of a branch
-	CommitURLTemplate string
-
-	// The repository URL for getting to a file.  Has access to
-	// {{Branch}}, {{Path}}
-	FileURLTemplate string
-
-	// The URL fragment to add to a file URL for line numbers. has
-	// access to {{LineNumber}}. The fragment should include the
-	// separator, generally '#' or ';'.
-	LineFragmentTemplate string
-
-	// All zoekt.* configuration settings.
-	RawConfig map[string]string
-
-	// Importance of the repository, bigger is more important
-	Rank uint16
-
-	// IndexOptions is a hash of the options used to create the index for the
-	// repo.
-	IndexOptions string
-}
-
-// IndexMetadata holds metadata stored in the index file. It contains
-// data generated by the core indexing library.
-type IndexMetadata struct {
-	IndexFormatVersion    int
-	IndexFeatureVersion   int
-	IndexMinReaderVersion int
-	IndexTime             time.Time
-	PlainASCII            bool
-	LanguageMap           map[string]byte
-	ZoektVersion          string
-}
-
-// Statistics of a (collection of) repositories.
-type RepoStats struct {
-	// Repos is used for aggregrating the number of repositories.
-	Repos int
-
-	// Shards is the total number of search shards.
-	Shards int
-
-	// Documents holds the number of documents or files.
-	Documents int
-
-	// IndexBytes is the amount of RAM used for index overhead.
-	IndexBytes int64
-
-	// ContentBytes is the amount of RAM used for raw content.
-	ContentBytes int64
-}
-
-func (s *RepoStats) Add(o *RepoStats) {
-	// can't update Repos, since one repo may have multiple
-	// shards.
-	s.Shards += o.Shards
-	s.IndexBytes += o.IndexBytes
-	s.Documents += o.Documents
-	s.ContentBytes += o.ContentBytes
-}
-
-type RepoListEntry struct {
-	Repository    Repository
-	IndexMetadata IndexMetadata
-	Stats         RepoStats
-}
-
-// RepoList holds a set of Repository metadata.
-type RepoList struct {
-	Repos   []*RepoListEntry
-	Crashes int
-}
-
-type Searcher interface {
-	Search(ctx context.Context, q query.Q, opts *SearchOptions) (*SearchResult, error)
-
-	// List lists repositories. The query `q` can only contain
-	// query.Repo atoms.
-	List(ctx context.Context, q query.Q) (*RepoList, error)
-	Close()
-
-	// Describe the searcher for debug messages.
-	String() string
-}
-
-type SearchOptions struct {
-	// Return an upper-bound estimate of eligible documents in
-	// stats.ShardFilesConsidered.
-	EstimateDocCount bool
-
-	// Return the whole file.
-	Whole bool
-
-	// Maximum number of matches: skip all processing an index
-	// shard after we found this many non-overlapping matches.
-	ShardMaxMatchCount int
-
-	// Maximum number of matches: stop looking for more matches
-	// once we have this many matches across shards.
-	TotalMaxMatchCount int
-
-	// Maximum number of important matches: skip processing
-	// shard after we found this many important matches.
-	ShardMaxImportantMatch int
-
-	// Maximum number of important matches across shards.
-	TotalMaxImportantMatch int
-
-	// Abort the search after this much time has passed.
-	MaxWallTime time.Duration
-
-	// Trim the number of results after collating and sorting the
-	// results
-	MaxDocDisplayCount int
-}
-
-func (s *SearchOptions) String() string {
-	return fmt.Sprintf("%#v", s)
-}
diff --git a/bits.go b/bits.go
deleted file mode 100644
index 62f6110..0000000
--- a/bits.go
+++ /dev/null
@@ -1,268 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"encoding/binary"
-	"unicode"
-	"unicode/utf8"
-)
-
-func generateCaseNgrams(g ngram) []ngram {
-	asRunes := ngramToRunes(g)
-
-	variants := make([]ngram, 0, 8)
-	cur := asRunes
-	for {
-		for i := 0; i < 3; i++ {
-			next := unicode.SimpleFold(cur[i])
-			cur[i] = next
-			if next != asRunes[i] {
-				break
-			}
-		}
-
-		variants = append(variants, runesToNGram(cur))
-		if cur == asRunes {
-			break
-		}
-	}
-
-	return variants
-}
-
-func toLower(in []byte) []byte {
-	out := make([]byte, 0, len(in))
-	var buf [4]byte
-	for _, c := range string(in) {
-		i := utf8.EncodeRune(buf[:], unicode.ToLower(c))
-		out = append(out, buf[:i]...)
-	}
-	return out
-}
-
-// compare 'lower' and 'mixed', where lower is the needle. 'mixed' may
-// be larger than 'lower'. Returns whether there was a match, and if
-// yes, the byte size of the match.
-func caseFoldingEqualsRunes(lower, mixed []byte) (int, bool) {
-	matchTotal := 0
-	for len(lower) > 0 && len(mixed) > 0 {
-		lr, lsz := utf8.DecodeRune(lower)
-		lower = lower[lsz:]
-
-		mr, msz := utf8.DecodeRune(mixed)
-		mixed = mixed[msz:]
-		matchTotal += msz
-
-		if lr != unicode.ToLower(mr) {
-			return 0, false
-		}
-	}
-
-	return matchTotal, len(lower) == 0
-}
-
-type ngram uint64
-
-func runesToNGram(b [ngramSize]rune) ngram {
-	return ngram(uint64(b[0])<<42 | uint64(b[1])<<21 | uint64(b[2]))
-}
-
-func bytesToNGram(b []byte) ngram {
-	return runesToNGram([ngramSize]rune{rune(b[0]), rune(b[1]), rune(b[2])})
-}
-
-func stringToNGram(s string) ngram {
-	return bytesToNGram([]byte(s))
-}
-
-func ngramToBytes(n ngram) []byte {
-	rs := ngramToRunes(n)
-	return []byte{byte(rs[0]), byte(rs[1]), byte(rs[2])}
-}
-
-const runeMask = 1<<21 - 1
-
-func ngramToRunes(n ngram) [ngramSize]rune {
-	return [ngramSize]rune{rune((n >> 42) & runeMask), rune((n >> 21) & runeMask), rune(n & runeMask)}
-}
-
-func (n ngram) String() string {
-	rs := ngramToRunes(n)
-	return string(rs[:])
-}
-
-type runeNgramOff struct {
-	ngram    ngram
-	byteSize uint32 // size of ngram
-	byteOff  uint32
-	runeOff  uint32
-}
-
-func splitNGrams(str []byte) []runeNgramOff {
-	var runeGram [3]rune
-	var off [3]uint32
-	var runeCount int
-
-	result := make([]runeNgramOff, 0, len(str))
-	var i uint32
-
-	chars := -1
-	for len(str) > 0 {
-		chars++
-		r, sz := utf8.DecodeRune(str)
-		str = str[sz:]
-		runeGram[0] = runeGram[1]
-		off[0] = off[1]
-		runeGram[1] = runeGram[2]
-		off[1] = off[2]
-		runeGram[2] = r
-		off[2] = uint32(i)
-		i += uint32(sz)
-		runeCount++
-		if runeCount < ngramSize {
-			continue
-		}
-
-		ng := runesToNGram(runeGram)
-		result = append(result, runeNgramOff{
-			ngram:    ng,
-			byteSize: i - off[0],
-			byteOff:  off[0],
-			runeOff:  uint32(chars),
-		})
-	}
-	return result
-}
-
-const (
-	_classChar  = 0
-	_classDigit = iota
-	_classPunct = iota
-	_classOther = iota
-	_classSpace = iota
-)
-
-func byteClass(c byte) int {
-	if (c >= 'a' && c <= 'z') || c >= 'A' && c <= 'Z' {
-		return _classChar
-	}
-	if c >= '0' && c <= '9' {
-		return _classDigit
-	}
-
-	switch c {
-	case ' ', '\n':
-		return _classSpace
-	case '.', ',', ';', '"', '\'':
-		return _classPunct
-	default:
-		return _classOther
-	}
-}
-
-func marshalDocSections(secs []DocumentSection) []byte {
-	ints := make([]uint32, 0, len(secs)*2)
-	for _, s := range secs {
-		ints = append(ints, uint32(s.Start), uint32(s.End))
-	}
-
-	return toSizedDeltas(ints)
-}
-
-func unmarshalDocSections(in []byte, buf []DocumentSection) (secs []DocumentSection) {
-	// TODO - ints is unnecessary garbage here.
-	ints := fromSizedDeltas(in, nil)
-	if cap(buf) >= len(ints)/2 {
-		buf = buf[:0]
-	} else {
-		buf = make([]DocumentSection, 0, len(ints)/2)
-	}
-
-	for len(ints) > 0 {
-		buf = append(buf, DocumentSection{ints[0], ints[1]})
-		ints = ints[2:]
-	}
-	return buf
-}
-
-type ngramSlice []ngram
-
-func (p ngramSlice) Len() int { return len(p) }
-
-func (p ngramSlice) Less(i, j int) bool {
-	return p[i] < p[j]
-}
-
-func (p ngramSlice) Swap(i, j int) {
-	p[i], p[j] = p[j], p[i]
-}
-
-func toSizedDeltas(offsets []uint32) []byte {
-	var enc [8]byte
-
-	deltas := make([]byte, 0, len(offsets)*2)
-
-	m := binary.PutUvarint(enc[:], uint64(len(offsets)))
-	deltas = append(deltas, enc[:m]...)
-
-	var last uint32
-	for _, p := range offsets {
-		delta := p - last
-		last = p
-
-		m := binary.PutUvarint(enc[:], uint64(delta))
-		deltas = append(deltas, enc[:m]...)
-	}
-	return deltas
-}
-
-func fromSizedDeltas(data []byte, ps []uint32) []uint32 {
-	sz, m := binary.Uvarint(data)
-	data = data[m:]
-
-	if cap(ps) < int(sz) {
-		ps = make([]uint32, 0, sz)
-	} else {
-		ps = ps[:0]
-	}
-
-	var last uint32
-	for len(data) > 0 {
-		delta, m := binary.Uvarint(data)
-		offset := last + uint32(delta)
-		last = offset
-		data = data[m:]
-		ps = append(ps, offset)
-	}
-	return ps
-}
-
-func fromDeltas(data []byte, buf []uint32) []uint32 {
-	buf = buf[:0]
-	if cap(buf) < len(data)/2 {
-		buf = make([]uint32, 0, len(data)/2)
-	}
-
-	var last uint32
-	for len(data) > 0 {
-		delta, m := binary.Uvarint(data)
-		offset := last + uint32(delta)
-		last = offset
-		data = data[m:]
-		buf = append(buf, offset)
-	}
-	return buf
-}
diff --git a/bits_test.go b/bits_test.go
deleted file mode 100644
index 41710a9..0000000
--- a/bits_test.go
+++ /dev/null
@@ -1,198 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"encoding/binary"
-	"log"
-	"math/rand"
-	"reflect"
-	"sort"
-	"testing"
-	"testing/quick"
-
-	"github.com/google/go-cmp/cmp"
-)
-
-var _ = log.Println
-
-func TestNgram(t *testing.T) {
-	in := "abc"
-	n := stringToNGram(in)
-	if n.String() != "abc" {
-		t.Errorf("got %q, want %q", n, "abc")
-	}
-
-	f := func(b ngramRunes) bool {
-		n := runesToNGram(b)
-		got := ngramRunes(ngramToRunes(n))
-		if !reflect.DeepEqual(b, got) {
-			t.Log(cmp.Diff(b, got))
-			return false
-		}
-		return true
-	}
-	if err := quick.Check(f, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-type ngramRunes [ngramSize]rune
-
-func (ngramRunes) Generate(rand *rand.Rand, size int) reflect.Value {
-	// Same implementation used by testing/quick to generate strings. But we
-	// force it to ngramSize runes.
-	var b ngramRunes
-	for i := range b {
-		b[i] = rune(rand.Intn(0x10ffff))
-	}
-	return reflect.ValueOf(b)
-}
-
-func TestDocSection(t *testing.T) {
-	in := []DocumentSection{{1, 2}, {3, 4}}
-	serialized := marshalDocSections(in)
-	roundtrip := unmarshalDocSections(serialized, nil)
-	if !reflect.DeepEqual(in, roundtrip) {
-		t.Errorf("got %v, want %v", roundtrip, in)
-	}
-}
-
-func TestGenerateCaseNgrams(t *testing.T) {
-	ng := stringToNGram("aB1")
-	gotNG := generateCaseNgrams(ng)
-
-	got := map[string]bool{}
-	for _, n := range gotNG {
-		got[string(ngramToBytes(n))] = true
-	}
-
-	want := map[string]bool{
-		"aB1": true,
-		"AB1": true,
-		"ab1": true,
-		"Ab1": true,
-	}
-
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestNextFileIndex(t *testing.T) {
-	for _, tc := range []struct {
-		off, curFile uint32
-		ends         []uint32
-		want         uint32
-	}{
-		{maxUInt32, 0, []uint32{34}, 1},
-		{9, 0, []uint32{34}, 0},
-		{450, 0, []uint32{100, 200, 300, 400, 500, 600}, 4},
-	} {
-		got := nextFileIndex(tc.off, tc.curFile, tc.ends)
-		if got != tc.want {
-			t.Errorf("%v: got %d, want %d", tc, got, tc.want)
-		}
-	}
-}
-
-func TestSizedDeltas(t *testing.T) {
-	encode := func(nums []uint32) []byte {
-		return toSizedDeltas(nums)
-	}
-	decode := func(data []byte) []uint32 {
-		if len(data) == 0 {
-			return nil
-		}
-		return fromSizedDeltas(data, nil)
-	}
-	testIncreasingIntCoder(t, encode, decode)
-}
-
-func TestFromDeltas(t *testing.T) {
-	decode := func(data []byte) []uint32 {
-		if len(data) == 0 {
-			return nil
-		}
-		return fromDeltas(data, nil)
-	}
-	testIncreasingIntCoder(t, toDeltas, decode)
-}
-
-func TestCompressedPostingIterator(t *testing.T) {
-	decode := func(data []byte) []uint32 {
-		if len(data) == 0 {
-			return nil
-		}
-
-		var nums []uint32
-		i := newCompressedPostingIterator(data, stringToNGram("abc"))
-		for i.first() != maxUInt32 {
-			nums = append(nums, i.first())
-			i.next(i.first())
-		}
-		return nums
-	}
-	testIncreasingIntCoder(t, toDeltas, decode)
-}
-
-func toDeltas(offsets []uint32) []byte {
-	var enc [8]byte
-
-	deltas := make([]byte, 0, len(offsets)*2)
-
-	var last uint32
-	for _, p := range offsets {
-		delta := p - last
-		last = p
-
-		m := binary.PutUvarint(enc[:], uint64(delta))
-		deltas = append(deltas, enc[:m]...)
-	}
-	return deltas
-}
-
-func testIncreasingIntCoder(t *testing.T, encode func([]uint32) []byte, decode func([]byte) []uint32) {
-	f := func(nums []uint32) bool {
-		nums = sortedUnique(nums)
-		b := encode(nums)
-		got := decode(b)
-		if len(nums) == len(got) && len(nums) == 0 {
-			return true
-		}
-		if !reflect.DeepEqual(got, nums) {
-			t.Log(cmp.Diff(nums, got))
-			return false
-		}
-		return true
-	}
-	if err := quick.Check(f, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func sortedUnique(nums []uint32) []uint32 {
-	if len(nums) == 0 {
-		return nums
-	}
-	sort.Slice(nums, func(i, j int) bool { return nums[i] < nums[j] })
-	filtered := nums[:1]
-	for _, n := range nums[1:] {
-		if filtered[len(filtered)-1] != n {
-			filtered = append(filtered, n)
-		}
-	}
-	return filtered
-}
diff --git a/build-deploy.sh b/build-deploy.sh
deleted file mode 100644
index 7fb442f..0000000
--- a/build-deploy.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# this script packages up all the binaries, and a script (deploy.sh)
-# to twiddle with the server and the binaries
-
-set -ex
-
-# Put the date first so we can sort.
-if [[ -z "$VERSION" ]]; then
-  VERSION=$(date --iso-8601=minutes | tr -d ':' | sed 's|\+.*$||')
-  if [[ -d .git ]]; then
-    VERSION=${VERSION}-$(git show --pretty=format:%h -q)
-  fi
-fi
-
-set -u
-
-out=zoekt-${VERSION}
-mkdir -p ${out}
-
-for d in $(find cmd  -maxdepth 1 -type d)
-do
-  go build -tags netgo -ldflags "-X github.com/google/zoekt.Version=$VERSION"  -o ${out}/$(basename $d) github.com/google/zoekt/$d
-done
-
-cat <<EOF > ${out}/deploy.sh
-#!/bin/bash
-
-echo "Set the following in the environment."
-echo ""
-echo '  export PATH="'$PWD'/bin:$PATH'
-echo ""
-
-set -eux
-
-# Allow sandbox to create NS's
-sudo sh -c 'echo 1 > /proc/sys/kernel/unprivileged_userns_clone'
-
-# we mmap the entire index, but typically only want the file contents.
-sudo sh -c 'echo 1 >/proc/sys/vm/overcommit_memory'
-
-# allow bind to 80 and 443
-sudo setcap 'cap_net_bind_service=+ep' bin/zoekt-webserver
-
-EOF
-
-chmod 755 ${out}/*
-
-tar --owner=root --group=root -czf zoekt-deploy-${VERSION}.tar.gz ${out}/*
-
-rm -rf ${out}
diff --git a/build/builder.go b/build/builder.go
deleted file mode 100644
index 5461828..0000000
--- a/build/builder.go
+++ /dev/null
@@ -1,567 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// package build implements a more convenient interface for building
-// zoekt indices.
-package build
-
-import (
-	"crypto/sha1"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"reflect"
-	"regexp"
-	"runtime"
-	"runtime/pprof"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/bmatcuk/doublestar"
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/ctags"
-)
-
-var DefaultDir = filepath.Join(os.Getenv("HOME"), ".zoekt")
-
-// Branch describes a single branch version.
-type Branch struct {
-	Name    string
-	Version string
-}
-
-// Options sets options for the index building.
-type Options struct {
-	// IndexDir is a directory that holds *.zoekt index files.
-	IndexDir string
-
-	// SizeMax is the maximum file size
-	SizeMax int
-
-	// Parallelism is the maximum number of shards to index in parallel
-	Parallelism int
-
-	// ShardMax sets the maximum corpus size for a single shard
-	ShardMax int
-
-	// TrigramMax sets the maximum number of distinct trigrams per document.
-	TrigramMax int
-
-	// RepositoryDescription holds names and URLs for the repository.
-	RepositoryDescription zoekt.Repository
-
-	// SubRepositories is a path => sub repository map.
-	SubRepositories map[string]*zoekt.Repository
-
-	// Path to exuberant ctags binary to run
-	CTags string
-
-	// If set, ctags must succeed.
-	CTagsMustSucceed bool
-
-	// Write memory profiles to this file.
-	MemProfile string
-
-	// LargeFiles is a slice of glob patterns, including ** for any number
-	// of directories, where matching file paths should be indexed
-	// regardless of their size. The full pattern syntax is here:
-	// https://github.com/bmatcuk/doublestar/tree/v1#patterns.
-	LargeFiles []string
-}
-
-// HashOptions creates a hash of the options that affect an index.
-func (o *Options) HashOptions() string {
-	hasher := sha1.New()
-
-	hasher.Write([]byte(o.CTags))
-	hasher.Write([]byte(fmt.Sprintf("%t", o.CTagsMustSucceed)))
-	hasher.Write([]byte(fmt.Sprintf("%d", o.SizeMax)))
-	hasher.Write([]byte(fmt.Sprintf("%q", o.LargeFiles)))
-
-	return fmt.Sprintf("%x", hasher.Sum(nil))
-}
-
-type largeFilesFlag struct{ *Options }
-
-func (f largeFilesFlag) String() string {
-	// From flag.Value documentation:
-	//
-	// The flag package may call the String method with a zero-valued receiver,
-	// such as a nil pointer.
-	if f.Options == nil {
-		return ""
-	}
-	s := append([]string{""}, f.LargeFiles...)
-	return strings.Join(s, "-large_file ")
-}
-
-func (f largeFilesFlag) Set(value string) error {
-	f.LargeFiles = append(f.LargeFiles, value)
-	return nil
-}
-
-// Flags adds flags for build options to fs.
-func (o *Options) Flags(fs *flag.FlagSet) {
-	x := *o
-	x.SetDefaults()
-	fs.IntVar(&o.SizeMax, "file_limit", x.SizeMax, "maximum file size")
-	fs.IntVar(&o.TrigramMax, "max_trigram_count", x.TrigramMax, "maximum number of trigrams per document")
-	fs.IntVar(&o.ShardMax, "shard_limit", x.ShardMax, "maximum corpus size for a shard")
-	fs.IntVar(&o.Parallelism, "parallelism", x.Parallelism, "maximum number of parallel indexing processes.")
-	fs.StringVar(&o.IndexDir, "index", x.IndexDir, "directory for search indices")
-	fs.BoolVar(&o.CTagsMustSucceed, "require_ctags", x.CTagsMustSucceed, "If set, ctags calls must succeed.")
-	fs.Var(largeFilesFlag{o}, "large_file", "A glob pattern where matching files are to be index regardless of their size. You can add multiple patterns by setting this more than once.")
-}
-
-// Builder manages (parallel) creation of uniformly sized shards. The
-// builder buffers up documents until it collects enough documents and
-// then builds a shard and writes.
-type Builder struct {
-	opts     Options
-	throttle chan int
-
-	nextShardNum int
-	todo         []*zoekt.Document
-	size         int
-
-	parser ctags.Parser
-
-	building sync.WaitGroup
-
-	errMu      sync.Mutex
-	buildError error
-
-	// temp name => final name for finished shards. We only rename
-	// them once all shards succeed to avoid Frankstein corpuses.
-	finishedShards map[string]string
-}
-
-type finishedShard struct {
-	temp, final string
-}
-
-// SetDefaults sets reasonable default options.
-func (o *Options) SetDefaults() {
-	if o.CTags == "" {
-		ctags, err := exec.LookPath("universal-ctags")
-		if err == nil {
-			o.CTags = ctags
-		}
-	}
-
-	if o.CTags == "" {
-		ctags, err := exec.LookPath("ctags-exuberant")
-		if err == nil {
-			o.CTags = ctags
-		}
-	}
-	if o.Parallelism == 0 {
-		o.Parallelism = 4
-	}
-	if o.SizeMax == 0 {
-		o.SizeMax = 2 << 20
-	}
-	if o.ShardMax == 0 {
-		o.ShardMax = 100 << 20
-	}
-	if o.TrigramMax == 0 {
-		o.TrigramMax = 20000
-	}
-
-	if o.RepositoryDescription.Name == "" && o.RepositoryDescription.URL != "" {
-		parsed, _ := url.Parse(o.RepositoryDescription.URL)
-		if parsed != nil {
-			o.RepositoryDescription.Name = filepath.Join(parsed.Host, parsed.Path)
-		}
-	}
-}
-
-func hashString(s string) string {
-	h := sha1.New()
-	io.WriteString(h, s)
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-// ShardName returns the name the given index shard.
-func (o *Options) shardName(n int) string {
-	abs := url.QueryEscape(o.RepositoryDescription.Name)
-	if len(abs) > 200 {
-		abs = abs[:200] + hashString(abs)[:8]
-	}
-	return filepath.Join(o.IndexDir,
-		fmt.Sprintf("%s_v%d.%05d.zoekt", abs, zoekt.IndexFormatVersion, n))
-}
-
-// IncrementalSkipIndexing returns true if the index present on disk matches
-// the build options.
-func (o *Options) IncrementalSkipIndexing() bool {
-	fn := o.shardName(0)
-
-	f, err := os.Open(fn)
-	if err != nil {
-		return false
-	}
-
-	iFile, err := zoekt.NewIndexFile(f)
-	if err != nil {
-		return false
-	}
-	defer iFile.Close()
-
-	repo, index, err := zoekt.ReadMetadata(iFile)
-	if err != nil {
-		return false
-	}
-
-	if index.IndexFeatureVersion != zoekt.FeatureVersion {
-		return false
-	}
-
-	if repo.IndexOptions != o.HashOptions() {
-		return false
-	}
-
-	return reflect.DeepEqual(repo.Branches, o.RepositoryDescription.Branches)
-}
-
-// IgnoreSizeMax determines whether the max size should be ignored.
-func (o *Options) IgnoreSizeMax(name string) bool {
-	for _, pattern := range o.LargeFiles {
-		pattern = strings.TrimSpace(pattern)
-		m, _ := doublestar.PathMatch(pattern, name)
-		if m {
-			return true
-		}
-	}
-
-	return false
-}
-
-// NewBuilder creates a new Builder instance.
-func NewBuilder(opts Options) (*Builder, error) {
-	opts.SetDefaults()
-	if opts.RepositoryDescription.Name == "" {
-		return nil, fmt.Errorf("builder: must set Name")
-	}
-
-	b := &Builder{
-		opts:           opts,
-		throttle:       make(chan int, opts.Parallelism),
-		finishedShards: map[string]string{},
-	}
-
-	if b.opts.CTags == "" && b.opts.CTagsMustSucceed {
-		return nil, fmt.Errorf("ctags binary not found, but CTagsMustSucceed set")
-	}
-
-	if strings.Contains(opts.CTags, "universal-ctags") {
-		parser, err := ctags.NewParser(opts.CTags)
-		if err != nil && opts.CTagsMustSucceed {
-			return nil, fmt.Errorf("ctags.NewParser: %v", err)
-		}
-
-		b.parser = parser
-	}
-	if _, err := b.newShardBuilder(); err != nil {
-		return nil, err
-	}
-
-	return b, nil
-}
-
-// AddFile is a convenience wrapper for the Add method
-func (b *Builder) AddFile(name string, content []byte) error {
-	return b.Add(zoekt.Document{Name: name, Content: content})
-}
-
-func (b *Builder) Add(doc zoekt.Document) error {
-	// We could pass the document on to the shardbuilder, but if
-	// we pass through a part of the source tree with binary/large
-	// files, the corresponding shard would be mostly empty, so
-	// insert a reason here too.
-	if len(doc.Content) > b.opts.SizeMax && !b.opts.IgnoreSizeMax(doc.Name) {
-		doc.SkipReason = fmt.Sprintf("document size %d larger than limit %d", len(doc.Content), b.opts.SizeMax)
-	} else if err := zoekt.CheckText(doc.Content, b.opts.TrigramMax); err != nil {
-		doc.SkipReason = err.Error()
-		doc.Language = "binary"
-	}
-
-	b.todo = append(b.todo, &doc)
-	b.size += len(doc.Name) + len(doc.Content)
-	if b.size > b.opts.ShardMax {
-		return b.flush()
-	}
-
-	return nil
-}
-
-// Finish creates a last shard from the buffered documents, and clears
-// stale shards from previous runs. This should always be called, also
-// in failure cases, to ensure cleanup.
-func (b *Builder) Finish() error {
-	b.flush()
-	b.building.Wait()
-
-	if b.buildError != nil {
-		for tmp := range b.finishedShards {
-			os.Remove(tmp)
-		}
-		b.finishedShards = map[string]string{}
-		return b.buildError
-	}
-
-	for tmp, final := range b.finishedShards {
-		if err := os.Rename(tmp, final); err != nil {
-			b.buildError = err
-		}
-	}
-	b.finishedShards = map[string]string{}
-
-	if b.nextShardNum > 0 {
-		b.deleteRemainingShards()
-	}
-	return b.buildError
-}
-
-func (b *Builder) deleteRemainingShards() {
-	for {
-		shard := b.nextShardNum
-		b.nextShardNum++
-		name := b.opts.shardName(shard)
-		if err := os.Remove(name); os.IsNotExist(err) {
-			break
-		}
-	}
-}
-
-func (b *Builder) flush() error {
-	todo := b.todo
-	b.todo = nil
-	b.size = 0
-	b.errMu.Lock()
-	defer b.errMu.Unlock()
-	if b.buildError != nil {
-		return b.buildError
-	}
-
-	hasShard := b.nextShardNum > 0
-	if len(todo) == 0 && hasShard {
-		return nil
-	}
-
-	shard := b.nextShardNum
-	b.nextShardNum++
-
-	if b.opts.Parallelism > 1 {
-		b.building.Add(1)
-		go func() {
-			b.throttle <- 1
-			done, err := b.buildShard(todo, shard)
-			<-b.throttle
-
-			b.errMu.Lock()
-			defer b.errMu.Unlock()
-			if err != nil && b.buildError == nil {
-				b.buildError = err
-			}
-			if err == nil {
-				b.finishedShards[done.temp] = done.final
-			}
-			b.building.Done()
-		}()
-	} else {
-		// No goroutines when we're not parallel. This
-		// simplifies memory profiling.
-		done, err := b.buildShard(todo, shard)
-		b.buildError = err
-		if err == nil {
-			b.finishedShards[done.temp] = done.final
-		}
-		if b.opts.MemProfile != "" {
-			// drop memory, and profile.
-			todo = nil
-			b.writeMemProfile(b.opts.MemProfile)
-		}
-
-		return b.buildError
-	}
-
-	return nil
-}
-
-var profileNumber int
-
-func (b *Builder) writeMemProfile(name string) {
-	nm := fmt.Sprintf("%s.%d", name, profileNumber)
-	profileNumber++
-	f, err := os.Create(nm)
-	if err != nil {
-		log.Fatal("could not create memory profile: ", err)
-	}
-	runtime.GC() // get up-to-date statistics
-	if err := pprof.WriteHeapProfile(f); err != nil {
-		log.Fatal("could not write memory profile: ", err)
-	}
-	f.Close()
-	log.Printf("wrote mem profile %q", nm)
-}
-
-// map [0,inf) to [0,1) monotonically
-func squashRange(j int) float64 {
-	x := float64(j)
-	return x / (1 + x)
-}
-
-var testRe = regexp.MustCompile("test")
-
-type rankedDoc struct {
-	*zoekt.Document
-	rank []float64
-}
-
-func rank(d *zoekt.Document, origIdx int) []float64 {
-	test := 0.0
-	if testRe.MatchString(d.Name) {
-		test = 1.0
-	}
-
-	// Smaller is earlier (=better).
-	return []float64{
-		// Prefer docs that are not tests
-		test,
-
-		// With many symbols
-		1.0 - squashRange(len(d.Symbols)),
-
-		// With short content
-		squashRange(len(d.Content)),
-
-		// With short names
-		squashRange(len(d.Name)),
-
-		// That is present is as many branches as possible
-		1.0 - squashRange(len(d.Branches)),
-
-		// Preserve original ordering.
-		squashRange(origIdx),
-	}
-}
-
-func sortDocuments(todo []*zoekt.Document) {
-	rs := make([]rankedDoc, 0, len(todo))
-	for i, t := range todo {
-		rd := rankedDoc{t, rank(t, i)}
-		rs = append(rs, rd)
-	}
-	sort.Slice(rs, func(i, j int) bool {
-		r1 := rs[i].rank
-		r2 := rs[j].rank
-		for i := range r1 {
-			if r1[i] < r2[i] {
-				return true
-			}
-			if r1[i] > r2[i] {
-				return false
-			}
-		}
-
-		return false
-	})
-	for i := range todo {
-		todo[i] = rs[i].Document
-	}
-}
-
-func (b *Builder) buildShard(todo []*zoekt.Document, nextShardNum int) (*finishedShard, error) {
-	if b.opts.CTags != "" {
-		err := ctagsAddSymbols(todo, b.parser, b.opts.CTags)
-		if b.opts.CTagsMustSucceed && err != nil {
-			return nil, err
-		}
-		if err != nil {
-			log.Printf("ignoring %s error: %v", b.opts.CTags, err)
-		}
-	}
-
-	name := b.opts.shardName(nextShardNum)
-
-	shardBuilder, err := b.newShardBuilder()
-	if err != nil {
-		return nil, err
-	}
-	sortDocuments(todo)
-	for _, t := range todo {
-		if err := shardBuilder.Add(*t); err != nil {
-			return nil, err
-		}
-	}
-
-	return b.writeShard(name, shardBuilder)
-}
-
-func (b *Builder) newShardBuilder() (*zoekt.IndexBuilder, error) {
-	desc := b.opts.RepositoryDescription
-	desc.SubRepoMap = b.opts.SubRepositories
-	desc.IndexOptions = b.opts.HashOptions()
-
-	shardBuilder, err := zoekt.NewIndexBuilder(&desc)
-	if err != nil {
-		return nil, err
-	}
-	return shardBuilder, nil
-}
-
-func (b *Builder) writeShard(fn string, ib *zoekt.IndexBuilder) (*finishedShard, error) {
-	dir := filepath.Dir(fn)
-	if err := os.MkdirAll(dir, 0o700); err != nil {
-		return nil, err
-	}
-
-	f, err := ioutil.TempFile(dir, filepath.Base(fn)+".*.tmp")
-	if err != nil {
-		return nil, err
-	}
-	if runtime.GOOS != "windows" {
-		if err := f.Chmod(0o666 &^ umask); err != nil {
-			return nil, err
-		}
-	}
-
-	defer f.Close()
-	if err := ib.Write(f); err != nil {
-		return nil, err
-	}
-	fi, err := f.Stat()
-	if err != nil {
-		return nil, err
-	}
-	if err := f.Close(); err != nil {
-		return nil, err
-	}
-
-	log.Printf("finished %s: %d index bytes (overhead %3.1f)", fn, fi.Size(),
-		float64(fi.Size())/float64(ib.ContentSize()+1))
-
-	return &finishedShard{f.Name(), fn}, nil
-}
-
-// umask holds the Umask of the current process
-var umask os.FileMode
diff --git a/build/builder_test.go b/build/builder_test.go
deleted file mode 100644
index 133b39a..0000000
--- a/build/builder_test.go
+++ /dev/null
@@ -1,51 +0,0 @@
-package build
-
-import (
-	"flag"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-)
-
-func TestFlags(t *testing.T) {
-	cases := []struct {
-		args []string
-		want Options
-	}{{
-		// Defaults
-		args: []string{},
-		want: Options{},
-	}, {
-		args: []string{"-index", "/tmp"},
-		want: Options{
-			IndexDir: "/tmp",
-		},
-	}, {
-		// single large file pattern
-		args: []string{"-large_file", "*.md"},
-		want: Options{
-			LargeFiles: []string{"*.md"},
-		},
-	}, {
-		// multiple large file pattern
-		args: []string{"-large_file", "*.md", "-large_file", "*.yaml"},
-		want: Options{
-			LargeFiles: []string{"*.md", "*.yaml"},
-		},
-	}}
-
-	for _, c := range cases {
-		c.want.SetDefaults()
-		// depends on $PATH setting.
-		c.want.CTags = ""
-
-		got := Options{}
-		fs := flag.NewFlagSet("", flag.ContinueOnError)
-		got.Flags(fs)
-		if err := fs.Parse(c.args); err != nil {
-			t.Errorf("failed to parse args %v: %v", c.args, err)
-		} else if !cmp.Equal(got, c.want) {
-			t.Errorf("mismatch for %v (-want +got):\n%s", c.args, cmp.Diff(c.want, got))
-		}
-	}
-}
diff --git a/build/builder_unix.go b/build/builder_unix.go
deleted file mode 100644
index fde31d5..0000000
--- a/build/builder_unix.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-// +build !windows
-
-package build
-
-import (
-	"os"
-	"syscall"
-)
-
-func init() {
-	umask = os.FileMode(syscall.Umask(0))
-	syscall.Umask(int(umask))
-}
diff --git a/build/ctags.go b/build/ctags.go
deleted file mode 100644
index 9c2ac5b..0000000
--- a/build/ctags.go
+++ /dev/null
@@ -1,278 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package build
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/ctags"
-)
-
-func runCTags(bin string, inputs map[string][]byte) ([]*ctags.Entry, error) {
-	const debug = false
-	if len(inputs) == 0 {
-		return nil, nil
-	}
-	dir, err := ioutil.TempDir("", "ctags-input")
-	if err != nil {
-		return nil, err
-	}
-	if !debug {
-		defer os.RemoveAll(dir)
-	}
-
-	// --sort shells out to sort(1).
-	args := []string{bin, "-n", "-f", "-", "--sort=no"}
-
-	fileCount := 0
-	for n, c := range inputs {
-		if len(c) == 0 {
-			continue
-		}
-
-		full := filepath.Join(dir, n)
-		if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil {
-			return nil, err
-		}
-		err := ioutil.WriteFile(full, c, 0o600)
-		if err != nil {
-			return nil, err
-		}
-		args = append(args, n)
-		fileCount++
-	}
-	if fileCount == 0 {
-		return nil, nil
-	}
-
-	cmd := exec.Command(args[0], args[1:]...)
-	cmd.Dir = dir
-
-	var errBuf, outBuf bytes.Buffer
-	cmd.Stderr = &errBuf
-	cmd.Stdout = &outBuf
-
-	if err := cmd.Start(); err != nil {
-		return nil, err
-	}
-
-	errChan := make(chan error, 1)
-	go func() {
-		err := cmd.Wait()
-		errChan <- err
-	}()
-	timeout := time.After(5 * time.Second)
-	select {
-	case <-timeout:
-		cmd.Process.Kill()
-		return nil, fmt.Errorf("timeout executing ctags")
-	case err := <-errChan:
-		if err != nil {
-			return nil, fmt.Errorf("exec(%s): %v, stderr: %s", cmd.Args, err, errBuf.String())
-		}
-	}
-
-	var entries []*ctags.Entry
-	for _, l := range bytes.Split(outBuf.Bytes(), []byte{'\n'}) {
-		if len(l) == 0 {
-			continue
-		}
-		e, err := ctags.Parse(string(l))
-		if err != nil {
-			return nil, err
-		}
-
-		if len(e.Sym) == 1 {
-			continue
-		}
-		entries = append(entries, e)
-	}
-	return entries, nil
-}
-
-func runCTagsChunked(bin string, in map[string][]byte) ([]*ctags.Entry, error) {
-	var res []*ctags.Entry
-
-	cur := map[string][]byte{}
-	sz := 0
-	for k, v := range in {
-		cur[k] = v
-		sz += len(k)
-
-		// 100k seems reasonable.
-		if sz > (100 << 10) {
-			r, err := runCTags(bin, cur)
-			if err != nil {
-				return nil, err
-			}
-			res = append(res, r...)
-
-			cur = map[string][]byte{}
-			sz = 0
-		}
-	}
-	r, err := runCTags(bin, cur)
-	if err != nil {
-		return nil, err
-	}
-	res = append(res, r...)
-	return res, nil
-}
-
-func ctagsAddSymbolsParser(todo []*zoekt.Document, parser ctags.Parser) error {
-	for _, doc := range todo {
-		if doc.Symbols != nil {
-			continue
-		}
-
-		es, err := parser.Parse(doc.Name, doc.Content)
-		if err != nil {
-			return err
-		}
-		if len(es) == 0 {
-			continue
-		}
-		doc.Language = strings.ToLower(es[0].Language)
-
-		symOffsets, err := tagsToSections(doc.Content, es)
-		if err != nil {
-			return fmt.Errorf("%s: %v", doc.Name, err)
-		}
-		doc.Symbols = symOffsets
-	}
-
-	return nil
-}
-
-func ctagsAddSymbols(todo []*zoekt.Document, parser ctags.Parser, bin string) error {
-	if parser != nil {
-		return ctagsAddSymbolsParser(todo, parser)
-	}
-
-	pathIndices := map[string]int{}
-	contents := map[string][]byte{}
-	for i, t := range todo {
-		if t.Symbols != nil {
-			continue
-		}
-
-		_, ok := pathIndices[t.Name]
-		if ok {
-			continue
-		}
-
-		pathIndices[t.Name] = i
-		contents[t.Name] = t.Content
-	}
-
-	var err error
-	var entries []*ctags.Entry
-	entries, err = runCTagsChunked(bin, contents)
-	if err != nil {
-		return err
-	}
-
-	fileTags := map[string][]*ctags.Entry{}
-	for _, e := range entries {
-		fileTags[e.Path] = append(fileTags[e.Path], e)
-	}
-
-	for k, tags := range fileTags {
-		symOffsets, err := tagsToSections(contents[k], tags)
-		if err != nil {
-			return fmt.Errorf("%s: %v", k, err)
-		}
-		todo[pathIndices[k]].Symbols = symOffsets
-		if len(tags) > 0 {
-			todo[pathIndices[k]].Language = strings.ToLower(tags[0].Language)
-		}
-	}
-	return nil
-}
-
-func tagsToSections(content []byte, tags []*ctags.Entry) ([]zoekt.DocumentSection, error) {
-	nls := newLinesIndices(content)
-	nls = append(nls, uint32(len(content)))
-	var symOffsets []zoekt.DocumentSection
-	var lastEnd uint32
-	var lastLine int
-	var lastIntraEnd int
-	for _, t := range tags {
-		if t.Line <= 0 {
-			// Observed this with a .JS file.
-			continue
-		}
-		lineIdx := t.Line - 1
-		if lineIdx >= len(nls) {
-			return nil, fmt.Errorf("linenum for entry out of range %v", t)
-		}
-
-		lineOff := uint32(0)
-		if lineIdx > 0 {
-			lineOff = nls[lineIdx-1] + 1
-		}
-
-		end := nls[lineIdx]
-		line := content[lineOff:end]
-		if lastLine == lineIdx {
-			line = line[lastIntraEnd:]
-		} else {
-			lastIntraEnd = 0
-		}
-
-		intraOff := lastIntraEnd + bytes.Index(line, []byte(t.Sym))
-		if intraOff < 0 {
-			// for Go code, this is very common, since
-			// ctags barfs on multi-line declarations
-			continue
-		}
-		start := lineOff + uint32(intraOff)
-		if start < lastEnd {
-			// This can happen if we have multiple tags on the same line.
-			// Give up.
-			continue
-		}
-
-		endSym := start + uint32(len(t.Sym))
-
-		symOffsets = append(symOffsets, zoekt.DocumentSection{
-			Start: start,
-			End:   endSym,
-		})
-		lastEnd = endSym
-		lastLine = lineIdx
-		lastIntraEnd = intraOff + len(t.Sym)
-	}
-
-	return symOffsets, nil
-}
-
-func newLinesIndices(in []byte) []uint32 {
-	out := make([]uint32, 0, len(in)/30)
-	for i, c := range in {
-		if c == '\n' {
-			out = append(out, uint32(i))
-		}
-	}
-	return out
-}
diff --git a/build/ctags_test.go b/build/ctags_test.go
deleted file mode 100644
index c853b2e..0000000
--- a/build/ctags_test.go
+++ /dev/null
@@ -1,94 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package build
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/ctags"
-)
-
-func TestTagsToSections(t *testing.T) {
-	c := []byte("package foo\nfunc bar(j int) {}\n//bla")
-	// ----------01234567890 1234567890123456789 012345
-
-	tags := []*ctags.Entry{
-		{
-			Sym:  "bar",
-			Line: 2,
-		},
-	}
-
-	secs, err := tagsToSections(c, tags)
-	if err != nil {
-		t.Fatal("tagsToSections", err)
-	}
-
-	if len(secs) != 1 || secs[0].Start != 17 || secs[0].End != 20 {
-		t.Fatalf("got %#v, want 1 section (17,20)", secs)
-	}
-}
-
-func TestTagsToSectionsMultiple(t *testing.T) {
-	c := []byte("class Foob { int x; int b; }")
-	// ----------012345678901234567890123456789
-
-	tags := []*ctags.Entry{
-		{
-			Sym:  "x",
-			Line: 1,
-		},
-		{
-			Sym:  "b",
-			Line: 1,
-		},
-	}
-
-	got, err := tagsToSections(c, tags)
-	if err != nil {
-		t.Fatal("tagsToSections", err)
-	}
-
-	want := []zoekt.DocumentSection{
-		{Start: 17, End: 18},
-		{Start: 24, End: 25},
-	}
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestTagsToSectionsEOF(t *testing.T) {
-	c := []byte("package foo\nfunc bar(j int) {}")
-	// ----------01234567890 1234567890123456789 012345
-
-	tags := []*ctags.Entry{
-		{
-			Sym:  "bar",
-			Line: 2,
-		},
-	}
-
-	secs, err := tagsToSections(c, tags)
-	if err != nil {
-		t.Fatal("tagsToSections", err)
-	}
-
-	if len(secs) != 1 || secs[0].Start != 17 || secs[0].End != 20 {
-		t.Fatalf("got %#v, want 1 section (17,20)", secs)
-	}
-}
diff --git a/build/e2e_test.go b/build/e2e_test.go
deleted file mode 100644
index d72cda4..0000000
--- a/build/e2e_test.go
+++ /dev/null
@@ -1,491 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package build
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"reflect"
-	"strings"
-	"testing"
-	"time"
-
-	"golang.org/x/net/context"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/query"
-	"github.com/google/zoekt/shards"
-)
-
-func TestBasic(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-
-	opts := Options{
-		IndexDir: dir,
-		ShardMax: 1024,
-		RepositoryDescription: zoekt.Repository{
-			Name: "repo",
-		},
-		Parallelism: 2,
-		SizeMax:     1 << 20,
-	}
-
-	b, err := NewBuilder(opts)
-	if err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	}
-
-	for i := 0; i < 4; i++ {
-		s := fmt.Sprintf("%d", i)
-		b.AddFile("F"+s, []byte(strings.Repeat(s, 1000)))
-	}
-
-	if err := b.Finish(); err != nil {
-		t.Errorf("Finish: %v", err)
-	}
-
-	fs, _ := filepath.Glob(dir + "/*")
-	if len(fs) <= 1 {
-		t.Fatalf("want multiple shards, got %v", fs)
-	}
-
-	ss, err := shards.NewDirectorySearcher(dir)
-	if err != nil {
-		t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
-	}
-
-	q, err := query.Parse("111")
-	if err != nil {
-		t.Fatalf("Parse(111): %v", err)
-	}
-
-	var sOpts zoekt.SearchOptions
-	ctx := context.Background()
-	result, err := ss.Search(ctx, q, &sOpts)
-	if err != nil {
-		t.Fatalf("Search(%v): %v", q, err)
-	}
-
-	if len(result.Files) != 1 || result.Files[0].FileName != "F1" {
-		t.Errorf("got %v, want 1 file.", result.Files)
-	}
-	defer ss.Close()
-}
-
-func TestLargeFileOption(t *testing.T) {
-	dir, err := ioutil.TempDir("", "large_files_test")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	sizeMax := 1000
-	opts := Options{
-		IndexDir:   dir,
-		LargeFiles: []string{"F0", "F2"},
-		RepositoryDescription: zoekt.Repository{
-			Name: "repo",
-		},
-		SizeMax: sizeMax,
-	}
-
-	b, err := NewBuilder(opts)
-	if err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	}
-
-	for i := 0; i < 4; i++ {
-		s := fmt.Sprintf("%d", i)
-		b.AddFile("F"+s, []byte(strings.Repeat("a", sizeMax+1)))
-	}
-
-	if err := b.Finish(); err != nil {
-		t.Errorf("Finish: %v", err)
-	}
-
-	ss, err := shards.NewDirectorySearcher(dir)
-	if err != nil {
-		t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
-	}
-
-	q, err := query.Parse("aaa")
-	if err != nil {
-		t.Fatalf("Parse(aaa): %v", err)
-	}
-
-	var sOpts zoekt.SearchOptions
-	ctx := context.Background()
-	result, err := ss.Search(ctx, q, &sOpts)
-	if err != nil {
-		t.Fatalf("Search(%v): %v", q, err)
-	}
-
-	if len(result.Files) != 2 {
-		t.Errorf("got %v files, want 2 files.", len(result.Files))
-	}
-	defer ss.Close()
-}
-
-func TestUpdate(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	opts := Options{
-		IndexDir: dir,
-		ShardMax: 1024,
-		RepositoryDescription: zoekt.Repository{
-			Name:            "repo",
-			FileURLTemplate: "url",
-		},
-		Parallelism: 2,
-		SizeMax:     1 << 20,
-	}
-
-	if b, err := NewBuilder(opts); err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	} else {
-		b.AddFile("F", []byte("hoi"))
-		if err := b.Finish(); err != nil {
-			t.Errorf("Finish: %v", err)
-		}
-	}
-	ss, err := shards.NewDirectorySearcher(dir)
-	if err != nil {
-		t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
-	}
-
-	ctx := context.Background()
-	repos, err := ss.List(ctx, &query.Repo{Pattern: "repo"})
-	if err != nil {
-		t.Fatalf("List: %v", err)
-	}
-
-	if len(repos.Repos) != 1 {
-		t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
-	}
-
-	fs, err := filepath.Glob(filepath.Join(dir, "*"))
-	if err != nil {
-		t.Fatalf("glob: %v", err)
-	}
-
-	opts.RepositoryDescription = zoekt.Repository{
-		Name:            "repo2",
-		FileURLTemplate: "url2",
-	}
-
-	if b, err := NewBuilder(opts); err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	} else {
-		b.AddFile("F", []byte("hoi"))
-		if err := b.Finish(); err != nil {
-			t.Errorf("Finish: %v", err)
-		}
-	}
-
-	// This is ugly, and potentially flaky, but there is no
-	// observable synchronization for the Sharded searcher, so
-	// this is the best we can do.
-	time.Sleep(100 * time.Millisecond)
-
-	ctx = context.Background()
-	if repos, err = ss.List(ctx, &query.Repo{Pattern: "repo"}); err != nil {
-		t.Fatalf("List: %v", err)
-	} else if len(repos.Repos) != 2 {
-		t.Errorf("List(repo): got %v, want 2 repos", repos.Repos)
-	}
-
-	for _, fn := range fs {
-		log.Printf("removing %s", fn)
-		if err := os.Remove(fn); err != nil {
-			t.Fatalf("Remove(%s): %v", fn, err)
-		}
-	}
-
-	time.Sleep(100 * time.Millisecond)
-
-	ctx = context.Background()
-	if repos, err = ss.List(ctx, &query.Repo{Pattern: "repo"}); err != nil {
-		t.Fatalf("List: %v", err)
-	} else if len(repos.Repos) != 1 {
-		var ss []string
-		for _, r := range repos.Repos {
-			ss = append(ss, r.Repository.Name)
-		}
-		t.Errorf("List(repo): got %v, want 1 repo", ss)
-	}
-}
-
-func TestDeleteOldShards(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	opts := Options{
-		IndexDir: dir,
-		ShardMax: 1024,
-		RepositoryDescription: zoekt.Repository{
-			Name:            "repo",
-			FileURLTemplate: "url",
-		},
-		SizeMax: 1 << 20,
-	}
-	opts.SetDefaults()
-
-	b, err := NewBuilder(opts)
-	if err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	}
-	for i := 0; i < 4; i++ {
-		s := fmt.Sprintf("%d\n", i)
-		b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2)))
-	}
-	if err := b.Finish(); err != nil {
-		t.Errorf("Finish: %v", err)
-	}
-
-	glob := filepath.Join(dir, "*")
-	fs, err := filepath.Glob(glob)
-	if err != nil {
-		t.Fatalf("Glob(%s): %v", glob, err)
-	} else if len(fs) != 4 {
-		t.Fatalf("Glob(%s): got %v, want 4 shards", glob, fs)
-	}
-
-	if fi, err := os.Lstat(fs[0]); err != nil {
-		t.Fatalf("Lstat: %v", err)
-	} else if fi.Mode()&0o666 == 0o600 {
-		// This fails spuriously if your umask is very restrictive.
-		t.Errorf("got mode %o, should respect umask.", fi.Mode())
-	}
-
-	// Do again, without sharding.
-	opts.ShardMax = 1 << 20
-	b, err = NewBuilder(opts)
-	if err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	}
-	for i := 0; i < 4; i++ {
-		s := fmt.Sprintf("%d\n", i)
-		b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2)))
-	}
-	if err := b.Finish(); err != nil {
-		t.Errorf("Finish: %v", err)
-	}
-
-	fs, err = filepath.Glob(glob)
-	if err != nil {
-		t.Fatalf("Glob(%s): %v", glob, err)
-	} else if len(fs) != 1 {
-		t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
-	}
-
-	// Again, but don't index anything; should leave old shards intact.
-	b, err = NewBuilder(opts)
-	if err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	}
-	if err := b.Finish(); err != nil {
-		t.Errorf("Finish: %v", err)
-	}
-
-	fs, err = filepath.Glob(glob)
-	if err != nil {
-		t.Fatalf("Glob(%s): %v", glob, err)
-	} else if len(fs) != 1 {
-		t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
-	}
-}
-
-func TestPartialSuccess(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	opts := Options{
-		IndexDir:    dir,
-		ShardMax:    1024,
-		SizeMax:     1 << 20,
-		Parallelism: 1,
-	}
-	opts.RepositoryDescription.Name = "repo"
-	opts.SetDefaults()
-
-	b, err := NewBuilder(opts)
-	if err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	}
-
-	for i := 0; i < 4; i++ {
-		nm := fmt.Sprintf("F%d", i)
-
-		// no error checking: the 2nd call will fail
-		b.AddFile(nm, []byte(strings.Repeat("01234567\n", 128)))
-		if i == 1 {
-			// force writes to fail.
-			if err := os.Chmod(dir, 0o555); err != nil {
-				t.Fatalf("chmod(%s): %s", dir, err)
-			}
-		}
-	}
-
-	if err := os.Chmod(dir, 0o755); err != nil {
-		t.Fatalf("chmod(%s, writable): %s", dir, err)
-	}
-
-	// No error checking.
-	b.Finish()
-
-	// Finish cleans up temporary files.
-	if fs, err := filepath.Glob(dir + "/*"); err != nil {
-		t.Errorf("glob(%s): %v", dir, err)
-	} else if len(fs) != 0 {
-		t.Errorf("got shards %v, want []", fs)
-	}
-}
-
-type filerankCase struct {
-	name string
-	docs []*zoekt.Document
-	want []int
-}
-
-func testFileRankAspect(t *testing.T, c filerankCase) {
-	var want []*zoekt.Document
-	for _, j := range c.want {
-		want = append(want, c.docs[j])
-	}
-
-	got := make([]*zoekt.Document, len(c.docs))
-	copy(got, c.docs)
-	sortDocuments(got)
-
-	print := func(ds []*zoekt.Document) string {
-		r := ""
-		for _, d := range ds {
-			r += fmt.Sprintf("%v, ", d)
-		}
-		return r
-	}
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got docs [%v], want [%v]", print(got), print(want))
-	}
-}
-
-func TestFileRank(t *testing.T) {
-	for _, c := range []filerankCase{{
-		name: "filename",
-		docs: []*zoekt.Document{
-			{
-				Name:    "longlonglong",
-				Content: []byte("bla"),
-			},
-			{
-				Name:    "short",
-				Content: []byte("bla"),
-			},
-		},
-		want: []int{1, 0},
-	}, {
-		name: "test",
-		docs: []*zoekt.Document{
-			{
-				Name:    "test",
-				Content: []byte("bla"),
-			},
-			{
-				Name:    "longlonglong",
-				Content: []byte("bla"),
-			},
-		},
-		want: []int{1, 0},
-	}, {
-		name: "content",
-		docs: []*zoekt.Document{
-			{
-				Content: []byte("bla"),
-			},
-			{
-				Content: []byte("blablablabla"),
-			},
-			{
-				Content: []byte("blabla"),
-			},
-		},
-		want: []int{0, 2, 1},
-	}} {
-		t.Run(c.name, func(t *testing.T) {
-			testFileRankAspect(t, c)
-		})
-	}
-}
-
-func TestEmptyContent(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	opts := Options{
-		IndexDir: dir,
-		RepositoryDescription: zoekt.Repository{
-			Name: "repo",
-		},
-	}
-	opts.SetDefaults()
-
-	b, err := NewBuilder(opts)
-	if err != nil {
-		t.Fatalf("NewBuilder: %v", err)
-	}
-	if err := b.Finish(); err != nil {
-		t.Errorf("Finish: %v", err)
-	}
-
-	fs, _ := filepath.Glob(dir + "/*")
-	if len(fs) != 1 {
-		t.Fatalf("want a shard, got %v", fs)
-	}
-
-	ss, err := shards.NewDirectorySearcher(dir)
-	if err != nil {
-		t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
-	}
-	defer ss.Close()
-
-	ctx := context.Background()
-	result, err := ss.List(ctx, &query.Const{Value: true})
-	if err != nil {
-		t.Fatalf("List: %v", err)
-	}
-
-	if len(result.Repos) != 1 || result.Repos[0].Repository.Name != "repo" {
-		t.Errorf("got %+v, want 1 repo.", result.Repos)
-	}
-}
diff --git a/cmd/flags.go b/cmd/flags.go
deleted file mode 100644
index e0b9005..0000000
--- a/cmd/flags.go
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2019 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-)
-
-var (
-	version = flag.Bool("version", false, "Print version number")
-	opts    = &build.Options{}
-)
-
-func init() {
-	opts.Flags(flag.CommandLine)
-}
-
-func OptionsFromFlags() *build.Options {
-	if *version {
-		name := filepath.Base(os.Args[0])
-		fmt.Printf("%s version %q\n", name, zoekt.Version)
-		os.Exit(0)
-	}
-
-	opts.SetDefaults()
-	return opts
-}
diff --git a/cmd/zoekt-archive-index/archive.go b/cmd/zoekt-archive-index/archive.go
deleted file mode 100644
index c1afe5b..0000000
--- a/cmd/zoekt-archive-index/archive.go
+++ /dev/null
@@ -1,188 +0,0 @@
-package main
-
-import (
-	"archive/tar"
-	"archive/zip"
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"os"
-	"strings"
-)
-
-type Archive interface {
-	Next() (*File, error)
-	Close() error
-}
-
-type File struct {
-	io.ReadCloser
-	Name string
-	Size int64
-}
-
-type tarArchive struct {
-	io.Closer
-	tr *tar.Reader
-}
-
-func (a *tarArchive) Next() (*File, error) {
-	for {
-		hdr, err := a.tr.Next()
-		if err != nil {
-			return nil, err
-		}
-
-		// We only care about files
-		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
-			continue
-		}
-
-		return &File{
-			ReadCloser: ioutil.NopCloser(a.tr),
-			Name:       hdr.Name,
-			Size:       hdr.Size,
-		}, nil
-	}
-}
-
-type zipArchive struct {
-	io.Closer
-	files []*zip.File
-}
-
-func (a *zipArchive) Next() (*File, error) {
-	if len(a.files) == 0 {
-		return nil, io.EOF
-	}
-
-	f := a.files[0]
-	a.files = a.files[1:]
-
-	r, err := f.Open()
-	if err != nil {
-		return nil, err
-	}
-
-	return &File{
-		ReadCloser: r,
-		Name:       f.Name,
-		Size:       int64(f.UncompressedSize64),
-	}, nil
-}
-
-func newZipArchive(r io.Reader, closer io.Closer) (*zipArchive, error) {
-	f, ok := r.(interface {
-		io.ReaderAt
-		Stat() (os.FileInfo, error)
-	})
-	if !ok {
-		return nil, fmt.Errorf("streaming zip files not supported")
-	}
-
-	fi, err := f.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	zr, err := zip.NewReader(f, fi.Size())
-	if err != nil {
-		return nil, err
-	}
-
-	// Filter out non files
-	files := zr.File[:0]
-	for _, f := range zr.File {
-		if f.Mode().IsRegular() {
-			files = append(files, f)
-		}
-	}
-
-	return &zipArchive{
-		Closer: closer,
-		files:  files,
-	}, nil
-}
-
-func detectContentType(r io.Reader) (string, io.Reader, error) {
-	var buf [512]byte
-	n, err := io.ReadFull(r, buf[:])
-	if err != nil && err != io.ErrUnexpectedEOF {
-		return "", nil, err
-	}
-
-	ct := http.DetectContentType(buf[:n])
-
-	// If we are a seeker, we can just undo our read
-	if s, ok := r.(io.Seeker); ok {
-		_, err := s.Seek(int64(-n), io.SeekCurrent)
-		return ct, r, err
-	}
-
-	// Otherwise return a new reader which merges in the read bytes
-	return ct, io.MultiReader(bytes.NewReader(buf[:n]), r), nil
-}
-
-func openReader(u string) (io.ReadCloser, error) {
-	if strings.HasPrefix(u, "https://") || strings.HasPrefix(u, "http://") {
-		resp, err := http.Get(u)
-		if err != nil {
-			return nil, err
-		}
-		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-			b, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1024))
-			_ = resp.Body.Close()
-			if err != nil {
-				return nil, err
-			}
-			return nil, &url.Error{
-				Op:  "Get",
-				URL: u,
-				Err: fmt.Errorf("%s: %s", resp.Status, string(b)),
-			}
-		}
-		return resp.Body, nil
-	} else if u == "-" {
-		return ioutil.NopCloser(os.Stdin), nil
-	}
-
-	return os.Open(u)
-}
-
-// openArchive opens the tar at the URL or filepath u. Also supported is tgz
-// files over http.
-func openArchive(u string) (ar Archive, err error) {
-	readCloser, err := openReader(u)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if err != nil {
-			_ = readCloser.Close()
-		}
-	}()
-
-	ct, r, err := detectContentType(readCloser)
-	if err != nil {
-		return nil, err
-	}
-	switch ct {
-	case "application/x-gzip":
-		r, err = gzip.NewReader(r)
-		if err != nil {
-			return nil, err
-		}
-
-	case "application/zip":
-		return newZipArchive(r, readCloser)
-	}
-
-	return &tarArchive{
-		Closer: readCloser,
-		tr:     tar.NewReader(r),
-	}, nil
-}
diff --git a/cmd/zoekt-archive-index/e2e_test.go b/cmd/zoekt-archive-index/e2e_test.go
deleted file mode 100644
index daaf556..0000000
--- a/cmd/zoekt-archive-index/e2e_test.go
+++ /dev/null
@@ -1,182 +0,0 @@
-package main
-
-import (
-	"archive/tar"
-	"archive/zip"
-	"compress/gzip"
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/query"
-	"github.com/google/zoekt/shards"
-)
-
-func TestMain(m *testing.M) {
-	flag.Parse()
-	if !testing.Verbose() {
-		log.SetOutput(ioutil.Discard)
-	}
-	os.Exit(m.Run())
-}
-
-func writeArchive(w io.Writer, format string, files map[string]string) (err error) {
-	if format == "zip" {
-		zw := zip.NewWriter(w)
-		for name, body := range files {
-			f, err := zw.Create(name)
-			if err != nil {
-				return err
-			}
-			if _, err := f.Write([]byte(body)); err != nil {
-				return err
-			}
-		}
-		return zw.Close()
-	}
-
-	if format == "tgz" {
-		gw := gzip.NewWriter(w)
-		defer func() {
-			err2 := gw.Close()
-			if err == nil {
-				err = err2
-			}
-		}()
-		w = gw
-		format = "tar"
-	}
-
-	if format != "tar" {
-		return errors.New("expected tar")
-	}
-
-	tw := tar.NewWriter(w)
-
-	for name, body := range files {
-		hdr := &tar.Header{
-			Name: name,
-			Mode: 0o600,
-			Size: int64(len(body)),
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-		if _, err := tw.Write([]byte(body)); err != nil {
-			return err
-		}
-	}
-	if err := tw.Close(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// TestIndexArg tests zoekt-archive-index by creating an archive and then
-// indexing and executing searches and checking we get expected results.
-// Additionally, we test that the index is properly updated with the
-// -incremental=true option changing the options between indexes and ensuring
-// the results change as expected.
-func TestIndexIncrementally(t *testing.T) {
-	for _, format := range []string{"tar", "tgz", "zip"} {
-		t.Run(format, func(t *testing.T) {
-			testIndexIncrementally(t, format)
-		})
-	}
-}
-
-func testIndexIncrementally(t *testing.T, format string) {
-	indexdir, err := ioutil.TempDir("", "TestIndexArg-index")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(indexdir)
-	archive, err := ioutil.TempFile("", "TestIndexArg-archive")
-	if err != nil {
-		t.Fatalf("TempFile: %v", err)
-	}
-	defer os.Remove(archive.Name())
-
-	fileSize := 1000
-
-	files := map[string]string{}
-	for i := 0; i < 4; i++ {
-		s := fmt.Sprintf("%d", i)
-		files["F"+s] = strings.Repeat("a", fileSize)
-	}
-
-	err = writeArchive(archive, format, files)
-	if err != nil {
-		t.Fatalf("unable to create archive %v", err)
-	}
-	archive.Close()
-
-	// tests contain options used to build an index and the expected number of
-	// files in the result set based on the options.
-	tests := []struct {
-		largeFiles   []string
-		wantNumFiles int
-	}{
-		{
-			largeFiles:   []string{},
-			wantNumFiles: 0,
-		},
-		{
-			largeFiles:   []string{"F0", "F2"},
-			wantNumFiles: 2,
-		},
-	}
-
-	for _, test := range tests {
-		largeFiles, wantNumFiles := test.largeFiles, test.wantNumFiles
-
-		bopts := build.Options{
-			SizeMax:    fileSize - 1,
-			IndexDir:   indexdir,
-			LargeFiles: largeFiles,
-		}
-		opts := Options{
-			Incremental: true,
-			Archive:     archive.Name(),
-			Name:        "repo",
-			Branch:      "master",
-			Commit:      "cccccccccccccccccccccccccccccccccccccccc",
-			Strip:       0,
-		}
-
-		if err := do(opts, bopts); err != nil {
-			t.Fatalf("error creating index: %v", err)
-		}
-
-		ss, err := shards.NewDirectorySearcher(indexdir)
-		if err != nil {
-			t.Fatalf("NewDirectorySearcher(%s): %v", indexdir, err)
-		}
-		defer ss.Close()
-
-		q, err := query.Parse("aaa")
-		if err != nil {
-			t.Fatalf("Parse(aaa): %v", err)
-		}
-
-		var sOpts zoekt.SearchOptions
-		result, err := ss.Search(context.Background(), q, &sOpts)
-		if err != nil {
-			t.Fatalf("Search(%v): %v", q, err)
-		}
-
-		if len(result.Files) != wantNumFiles {
-			t.Errorf("got %v, want %d files.", result.Files, wantNumFiles)
-		}
-	}
-}
diff --git a/cmd/zoekt-archive-index/main.go b/cmd/zoekt-archive-index/main.go
deleted file mode 100644
index 8754b4e..0000000
--- a/cmd/zoekt-archive-index/main.go
+++ /dev/null
@@ -1,232 +0,0 @@
-// Command zoekt-archive-index indexes an archive.
-//
-// Example via github.com:
-//
-//   zoekt-archive-index -incremental -commit b57cb1605fd11ba2ecfa7f68992b4b9cc791934d -name github.com/gorilla/mux -strip_components 1 https://codeload.github.com/gorilla/mux/legacy.tar.gz/b57cb1605fd11ba2ecfa7f68992b4b9cc791934d
-//
-//   zoekt-archive-index -branch master https://github.com/gorilla/mux/commit/b57cb1605fd11ba2ecfa7f68992b4b9cc791934d
-package main
-
-import (
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"net/url"
-	"strings"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/cmd"
-	"github.com/google/zoekt/gitindex"
-	"go.uber.org/automaxprocs/maxprocs"
-)
-
-// stripComponents removes the specified number of leading path
-// elements. Pathnames with fewer elements will return the empty string.
-func stripComponents(path string, count int) string {
-	for i := 0; path != "" && i < count; i++ {
-		i := strings.Index(path, "/")
-		if i < 0 {
-			return ""
-		}
-		path = path[i+1:]
-	}
-	return path
-}
-
-// isGitOID checks if the revision is a git OID SHA string.
-//
-// Note: This doesn't mean the SHA exists in a repository, nor does it mean it
-// isn't a ref. Git allows 40-char hexadecimal strings to be references.
-func isGitOID(s string) bool {
-	if len(s) != 40 {
-		return false
-	}
-	for _, r := range s {
-		if !(('0' <= r && r <= '9') ||
-			('a' <= r && r <= 'f') ||
-			('A' <= r && r <= 'F')) {
-			return false
-		}
-	}
-	return true
-}
-
-type Options struct {
-	Incremental bool
-
-	Archive string
-	Name    string
-	RepoURL string
-	Branch  string
-	Commit  string
-	Strip   int
-}
-
-func (o *Options) SetDefaults() {
-	// We guess based on the archive URL.
-	u, _ := url.Parse(o.Archive)
-	if u == nil {
-		return
-	}
-
-	setRef := func(ref string) {
-		if isGitOID(ref) && o.Commit == "" {
-			o.Commit = ref
-		}
-		if !isGitOID(ref) && o.Branch == "" {
-			o.Branch = ref
-		}
-	}
-
-	switch u.Host {
-	case "github.com", "codeload.github.com":
-		// https://github.com/octokit/octokit.rb/commit/3d21ec53a331a6f037a91c368710b99387d012c1
-		// https://github.com/octokit/octokit.rb/blob/master/README.md
-		// https://github.com/octokit/octokit.rb/tree/master/lib
-		// https://codeload.github.com/octokit/octokit.rb/legacy.tar.gz/master
-		parts := strings.Split(u.Path, "/")
-		if len(parts) > 2 && o.Name == "" {
-			o.Name = fmt.Sprintf("github.com/%s/%s", parts[1], parts[2])
-			o.RepoURL = fmt.Sprintf("https://github.com/%s/%s", parts[1], parts[2])
-		}
-		if len(parts) > 4 {
-			setRef(parts[4])
-			if u.Host == "github.com" {
-				o.Archive = fmt.Sprintf("https://codeload.github.com/%s/%s/legacy.tar.gz/%s", parts[1], parts[2], parts[4])
-			}
-		}
-		o.Strip = 1
-	case "api.github.com":
-		// https://api.github.com/repos/octokit/octokit.rb/tarball/master
-		parts := strings.Split(u.Path, "/")
-		if len(parts) > 2 && o.Name == "" {
-			o.Name = fmt.Sprintf("github.com/%s/%s", parts[1], parts[2])
-			o.RepoURL = fmt.Sprintf("https://github.com/%s/%s", parts[1], parts[2])
-		}
-		if len(parts) > 5 {
-			setRef(parts[5])
-		}
-		o.Strip = 1
-	}
-}
-
-func do(opts Options, bopts build.Options) error {
-	opts.SetDefaults()
-
-	if opts.Name == "" && opts.RepoURL == "" {
-		return errors.New("-name or -url required")
-	}
-	if opts.Branch == "" {
-		return errors.New("-branch required")
-	}
-
-	if opts.Name != "" {
-		bopts.RepositoryDescription.Name = opts.Name
-	}
-	if opts.RepoURL != "" {
-		u, err := url.Parse(opts.RepoURL)
-		if err != nil {
-			return err
-		}
-		if err := gitindex.SetTemplatesFromOrigin(&bopts.RepositoryDescription, u); err != nil {
-			return err
-		}
-	}
-	bopts.SetDefaults()
-	bopts.RepositoryDescription.Branches = []zoekt.RepositoryBranch{{Name: opts.Branch, Version: opts.Commit}}
-	brs := []string{opts.Branch}
-
-	if opts.Incremental && bopts.IncrementalSkipIndexing() {
-		return nil
-	}
-
-	a, err := openArchive(opts.Archive)
-	if err != nil {
-		return err
-	}
-	defer a.Close()
-
-	bopts.RepositoryDescription.Source = opts.Archive
-	builder, err := build.NewBuilder(bopts)
-	if err != nil {
-		return err
-	}
-
-	add := func(f *File) error {
-		defer f.Close()
-
-		contents, err := ioutil.ReadAll(f)
-		if err != nil {
-			return err
-		}
-
-		name := stripComponents(f.Name, opts.Strip)
-		if name == "" {
-			return nil
-		}
-
-		return builder.Add(zoekt.Document{
-			Name:     name,
-			Content:  contents,
-			Branches: brs,
-		})
-	}
-
-	for {
-		f, err := a.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return err
-		}
-
-		if err := add(f); err != nil {
-			return err
-		}
-	}
-
-	return builder.Finish()
-}
-
-func main() {
-	var (
-		incremental = flag.Bool("incremental", true, "only index changed repositories")
-
-		name   = flag.String("name", "", "The repository name for the archive")
-		urlRaw = flag.String("url", "", "The repository URL for the archive")
-		branch = flag.String("branch", "", "The branch name for the archive")
-		commit = flag.String("commit", "", "The commit sha for the archive. If incremental this will avoid updating shards already at commit")
-		strip  = flag.Int("strip_components", 0, "Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.")
-	)
-	flag.Parse()
-
-	// Tune GOMAXPROCS to match Linux container CPU quota.
-	maxprocs.Set()
-
-	log.SetFlags(log.LstdFlags | log.Lshortfile)
-
-	if len(flag.Args()) != 1 {
-		log.Fatal("expected argument for archive location")
-	}
-	archive := flag.Args()[0]
-	bopts := cmd.OptionsFromFlags()
-	opts := Options{
-		Incremental: *incremental,
-
-		Archive: archive,
-		Name:    *name,
-		RepoURL: *urlRaw,
-		Branch:  *branch,
-		Commit:  *commit,
-		Strip:   *strip,
-	}
-
-	if err := do(opts, *bopts); err != nil {
-		log.Fatal(err)
-	}
-}
diff --git a/cmd/zoekt-git-clone/main.go b/cmd/zoekt-git-clone/main.go
deleted file mode 100644
index eb85488..0000000
--- a/cmd/zoekt-git-clone/main.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary fetches all repos of a user or organization and clones
-// them.  It is strongly recommended to get a personal API token from
-// https://github.com/settings/tokens, save the token in a file, and
-// point the --token option to it.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/google/zoekt/gitindex"
-)
-
-func main() {
-	dest := flag.String("dest", "", "destination directory")
-	flag.Parse()
-
-	if *dest == "" {
-		log.Fatal("must set --dest")
-	}
-	if len(flag.Args()) == 0 {
-		log.Fatal("must provide URL")
-	}
-	u, err := url.Parse(flag.Arg(0))
-	if err != nil {
-		log.Fatalf("url.Parse: %v", err)
-	}
-
-	name := filepath.Join(u.Host, u.Path)
-	name = strings.TrimSuffix(name, ".git")
-
-	destDir := filepath.Dir(filepath.Join(*dest, name))
-	if err := os.MkdirAll(destDir, 0o755); err != nil {
-		log.Fatal(err)
-	}
-
-	config := map[string]string{
-		"zoekt.name": name,
-	}
-
-	destRepo, err := gitindex.CloneRepo(destDir, filepath.Base(name), u.String(), config)
-	if err != nil {
-		log.Fatalf("CloneRepo: %v", err)
-	}
-	if destRepo != "" {
-		fmt.Println(destRepo)
-	}
-}
diff --git a/cmd/zoekt-git-index/main.go b/cmd/zoekt-git-index/main.go
deleted file mode 100644
index 50997e5..0000000
--- a/cmd/zoekt-git-index/main.go
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/google/zoekt/cmd"
-	"github.com/google/zoekt/gitindex"
-	"go.uber.org/automaxprocs/maxprocs"
-)
-
-func main() {
-	allowMissing := flag.Bool("allow_missing_branches", false, "allow missing branches.")
-	submodules := flag.Bool("submodules", true, "if set to false, do not recurse into submodules")
-	branchesStr := flag.String("branches", "HEAD", "git branches to index.")
-	branchPrefix := flag.String("prefix", "refs/heads/", "prefix for branch names")
-
-	incremental := flag.Bool("incremental", true, "only index changed repositories")
-	repoCacheDir := flag.String("repo_cache", "", "directory holding bare git repos, named by URL. "+
-		"this is used to find repositories for submodules. "+
-		"It also affects name if the indexed repository is under this directory.")
-	flag.Parse()
-
-	// Tune GOMAXPROCS to match Linux container CPU quota.
-	maxprocs.Set()
-
-	if *repoCacheDir != "" {
-		dir, err := filepath.Abs(*repoCacheDir)
-		if err != nil {
-			log.Fatalf("Abs: %v", err)
-		}
-		*repoCacheDir = dir
-	}
-	opts := cmd.OptionsFromFlags()
-
-	var branches []string
-	if *branchesStr != "" {
-		branches = strings.Split(*branchesStr, ",")
-	}
-
-	gitRepos := map[string]string{}
-	for _, repoDir := range flag.Args() {
-		repoDir, err := filepath.Abs(repoDir)
-		if err != nil {
-			log.Fatal(err)
-		}
-		repoDir = filepath.Clean(repoDir)
-
-		name := strings.TrimSuffix(repoDir, "/.git")
-		if *repoCacheDir != "" && strings.HasPrefix(name, *repoCacheDir) {
-			name = strings.TrimPrefix(name, *repoCacheDir+"/")
-			name = strings.TrimSuffix(name, ".git")
-		} else {
-			name = strings.TrimSuffix(filepath.Base(name), ".git")
-		}
-		gitRepos[repoDir] = name
-	}
-
-	exitStatus := 0
-	for dir, name := range gitRepos {
-		opts.RepositoryDescription.Name = name
-		gitOpts := gitindex.Options{
-			BranchPrefix:       *branchPrefix,
-			Incremental:        *incremental,
-			Submodules:         *submodules,
-			RepoCacheDir:       *repoCacheDir,
-			AllowMissingBranch: *allowMissing,
-			BuildOptions:       *opts,
-			Branches:           branches,
-			RepoDir:            dir,
-		}
-
-		if err := gitindex.IndexGitRepo(gitOpts); err != nil {
-			log.Printf("indexGitRepo(%s): %v", dir, err)
-			exitStatus = 1
-		}
-	}
-	os.Exit(exitStatus)
-}
diff --git a/cmd/zoekt-hg-index/main.go b/cmd/zoekt-hg-index/main.go
deleted file mode 100644
index dd782a6..0000000
--- a/cmd/zoekt-hg-index/main.go
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2020 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// zoekt-hg-index provides bare-bones Mercurial indexing
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"path/filepath"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/cmd"
-
-	"go.uber.org/automaxprocs/maxprocs"
-	"humungus.tedunangst.com/r/gerc"
-)
-
-func main() {
-	revisionStr := flag.String("revision", "", "hg revision to index")
-	flag.Parse()
-	maxprocs.Set()
-	opts := cmd.OptionsFromFlags()
-
-	if len(flag.Args()) < 1 {
-		log.Fatal("hg repo directory argument missing")
-	}
-	dir, err := filepath.Abs(flag.Arg(0))
-	if err != nil {
-		log.Fatal(err)
-	}
-	opts.RepositoryDescription.Name = dir
-
-	if err := indexHg(dir, *revisionStr, opts); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func indexHg(dir, rev string, opts *build.Options) error {
-	r, err := gerc.Open(dir)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer r.Close()
-
-	builder, err := build.NewBuilder(*opts)
-	if err != nil {
-		return err
-	}
-	defer builder.Finish()
-
-	mfs, err := r.GetFiles(gerc.FilesArgs{
-		Revision: rev,
-	})
-	if err != nil {
-		return fmt.Errorf("GetFiles %v", err)
-	}
-
-	for _, mf := range mfs {
-		fd := gerc.FileDataArgs{
-			Filename: mf.Name,
-			Revision: rev,
-		}
-		content, err := r.GetFileData(fd)
-		if err != nil {
-			return fmt.Errorf("GetFileData %v", err)
-		}
-		if err := builder.Add(zoekt.Document{
-			Name:    mf.Name,
-			Content: content,
-		}); err != nil {
-			return err
-		}
-	}
-	return builder.Finish()
-}
diff --git a/cmd/zoekt-index/main.go b/cmd/zoekt-index/main.go
deleted file mode 100644
index a103f21..0000000
--- a/cmd/zoekt-index/main.go
+++ /dev/null
@@ -1,143 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"runtime/pprof"
-	"strings"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/cmd"
-	"go.uber.org/automaxprocs/maxprocs"
-)
-
-type fileInfo struct {
-	name string
-	size int64
-}
-
-type fileAggregator struct {
-	ignoreDirs map[string]struct{}
-	sizeMax    int64
-	sink       chan fileInfo
-}
-
-func (a *fileAggregator) add(path string, info os.FileInfo, err error) error {
-	if err != nil {
-		return err
-	}
-
-	if info.IsDir() {
-		base := filepath.Base(path)
-		if _, ok := a.ignoreDirs[base]; ok {
-			return filepath.SkipDir
-		}
-	}
-
-	if info.Mode().IsRegular() {
-		a.sink <- fileInfo{path, info.Size()}
-	}
-	return nil
-}
-
-func main() {
-	cpuProfile := flag.String("cpu_profile", "", "write cpu profile to file")
-	ignoreDirs := flag.String("ignore_dirs", ".git,.hg,.svn", "comma separated list of directories to ignore.")
-	flag.Parse()
-
-	// Tune GOMAXPROCS to match Linux container CPU quota.
-	maxprocs.Set()
-
-	opts := cmd.OptionsFromFlags()
-	if *cpuProfile != "" {
-		f, err := os.Create(*cpuProfile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-	}
-
-	ignoreDirMap := map[string]struct{}{}
-	if *ignoreDirs != "" {
-		dirs := strings.Split(*ignoreDirs, ",")
-		for _, d := range dirs {
-			d = strings.TrimSpace(d)
-			if d != "" {
-				ignoreDirMap[d] = struct{}{}
-			}
-		}
-	}
-	for _, arg := range flag.Args() {
-		opts.RepositoryDescription.Source = arg
-		if err := indexArg(arg, *opts, ignoreDirMap); err != nil {
-			log.Fatal(err)
-		}
-	}
-}
-
-func indexArg(arg string, opts build.Options, ignore map[string]struct{}) error {
-	dir, err := filepath.Abs(filepath.Clean(arg))
-	if err != nil {
-		return err
-	}
-
-	opts.RepositoryDescription.Name = filepath.Base(dir)
-	builder, err := build.NewBuilder(opts)
-	if err != nil {
-		return err
-	}
-	defer builder.Finish()
-
-	comm := make(chan fileInfo, 100)
-	agg := fileAggregator{
-		ignoreDirs: ignore,
-		sink:       comm,
-		sizeMax:    int64(opts.SizeMax),
-	}
-
-	go func() {
-		if err := filepath.Walk(dir, agg.add); err != nil {
-			log.Fatal(err)
-		}
-		close(comm)
-	}()
-
-	for f := range comm {
-		displayName := strings.TrimPrefix(f.name, dir+"/")
-		if f.size > int64(opts.SizeMax) && !opts.IgnoreSizeMax(displayName) {
-			builder.Add(zoekt.Document{
-				Name:       displayName,
-				SkipReason: fmt.Sprintf("document size %d larger than limit %d", f.size, opts.SizeMax),
-			})
-			continue
-		}
-		content, err := ioutil.ReadFile(f.name)
-		if err != nil {
-			return err
-		}
-
-		builder.AddFile(displayName, content)
-	}
-
-	return builder.Finish()
-}
diff --git a/cmd/zoekt-indexserver/config.go b/cmd/zoekt-indexserver/config.go
deleted file mode 100644
index d213774..0000000
--- a/cmd/zoekt-indexserver/config.go
+++ /dev/null
@@ -1,268 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"log"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-)
-
-type ConfigEntry struct {
-	GithubUser             string
-	GithubOrg              string
-	BitBucketServerProject string
-	GitHubURL              string
-	GitilesURL             string
-	CGitURL                string
-	BitBucketServerURL     string
-	DisableTLS             bool
-	CredentialPath         string
-	ProjectType            string
-	Name                   string
-	Exclude                string
-	GitLabURL              string
-	OnlyPublic             bool
-	GerritApiURL           string
-	Topics                 []string
-	ExcludeTopics          []string
-}
-
-func randomize(entries []ConfigEntry) []ConfigEntry {
-	perm := rand.Perm(len(entries))
-
-	var shuffled []ConfigEntry
-	for _, i := range perm {
-		shuffled = append(shuffled, entries[i])
-	}
-
-	return shuffled
-}
-
-func isHTTP(u string) bool {
-	asURL, err := url.Parse(u)
-	return err == nil && (asURL.Scheme == "http" || asURL.Scheme == "https")
-}
-
-func readConfigURL(u string) ([]ConfigEntry, error) {
-	var body []byte
-	var readErr error
-
-	if isHTTP(u) {
-		rep, err := http.Get(u)
-		if err != nil {
-			return nil, err
-		}
-		defer rep.Body.Close()
-
-		body, readErr = ioutil.ReadAll(rep.Body)
-	} else {
-		body, readErr = ioutil.ReadFile(u)
-	}
-
-	if readErr != nil {
-		return nil, readErr
-	}
-
-	var result []ConfigEntry
-	if err := json.Unmarshal(body, &result); err != nil {
-		return nil, err
-	}
-	return result, nil
-}
-
-func watchFile(path string) (<-chan struct{}, error) {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		return nil, err
-	}
-
-	if err := watcher.Add(filepath.Dir(path)); err != nil {
-		return nil, err
-	}
-
-	out := make(chan struct{}, 1)
-	go func() {
-		var last time.Time
-		for {
-			select {
-			case <-watcher.Events:
-				fi, err := os.Stat(path)
-				if err == nil && fi.ModTime() != last {
-					out <- struct{}{}
-					last = fi.ModTime()
-				}
-			case err := <-watcher.Errors:
-				if err != nil {
-					log.Printf("watcher error: %v", err)
-				}
-			}
-		}
-	}()
-	return out, nil
-}
-
-func periodicMirrorFile(repoDir string, opts *Options, pendingRepos chan<- string) {
-	ticker := time.NewTicker(opts.mirrorInterval)
-
-	var watcher <-chan struct{}
-	if !isHTTP(opts.mirrorConfigFile) {
-		var err error
-		watcher, err = watchFile(opts.mirrorConfigFile)
-		if err != nil {
-			log.Printf("watchFile(%q): %v", opts.mirrorConfigFile, err)
-		}
-	}
-
-	var lastCfg []ConfigEntry
-	for {
-		cfg, err := readConfigURL(opts.mirrorConfigFile)
-		if err != nil {
-			log.Printf("readConfig(%s): %v", opts.mirrorConfigFile, err)
-		} else {
-			lastCfg = cfg
-		}
-
-		executeMirror(lastCfg, repoDir, pendingRepos)
-
-		select {
-		case <-watcher:
-			log.Printf("mirror config %s changed", opts.mirrorConfigFile)
-		case <-ticker.C:
-		}
-	}
-}
-
-func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string) {
-	// Randomize the ordering in which we query
-	// things. This is to ensure that quota limits don't
-	// always hit the last one in the list.
-	cfg = randomize(cfg)
-	for _, c := range cfg {
-		var cmd *exec.Cmd
-		if c.GitHubURL != "" || c.GithubUser != "" || c.GithubOrg != "" {
-			cmd = exec.Command("zoekt-mirror-github",
-				"-dest", repoDir, "-delete")
-			if c.GitHubURL != "" {
-				cmd.Args = append(cmd.Args, "-url", c.GitHubURL)
-			}
-			if c.GithubUser != "" {
-				cmd.Args = append(cmd.Args, "-user", c.GithubUser)
-			} else if c.GithubOrg != "" {
-				cmd.Args = append(cmd.Args, "-org", c.GithubOrg)
-			}
-			if c.Name != "" {
-				cmd.Args = append(cmd.Args, "-name", c.Name)
-			}
-			if c.Exclude != "" {
-				cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
-			}
-			if c.CredentialPath != "" {
-				cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
-			}
-			for _, topic := range c.Topics {
-				cmd.Args = append(cmd.Args, "-topic", topic)
-			}
-			for _, topic := range c.ExcludeTopics {
-				cmd.Args = append(cmd.Args, "-exclude_topic", topic)
-			}
-		} else if c.GitilesURL != "" {
-			cmd = exec.Command("zoekt-mirror-gitiles",
-				"-dest", repoDir, "-name", c.Name)
-			if c.Exclude != "" {
-				cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
-			}
-			cmd.Args = append(cmd.Args, c.GitilesURL)
-		} else if c.CGitURL != "" {
-			cmd = exec.Command("zoekt-mirror-gitiles",
-				"-type", "cgit",
-				"-dest", repoDir, "-name", c.Name)
-			if c.Exclude != "" {
-				cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
-			}
-			cmd.Args = append(cmd.Args, c.CGitURL)
-		} else if c.BitBucketServerURL != "" {
-			cmd = exec.Command("zoekt-mirror-bitbucket-server",
-				"-dest", repoDir, "-url", c.BitBucketServerURL, "-delete")
-			if c.BitBucketServerProject != "" {
-				cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject)
-			}
-			if c.DisableTLS {
-				cmd.Args = append(cmd.Args, "-disable-tls")
-			}
-			if c.ProjectType != "" {
-				cmd.Args = append(cmd.Args, "-type", c.ProjectType)
-			}
-			if c.Name != "" {
-				cmd.Args = append(cmd.Args, "-name", c.Name)
-			}
-			if c.Exclude != "" {
-				cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
-			}
-			if c.CredentialPath != "" {
-				cmd.Args = append(cmd.Args, "-credentials", c.CredentialPath)
-			}
-		} else if c.GitLabURL != "" {
-			cmd = exec.Command("zoekt-mirror-gitlab",
-				"-dest", repoDir, "-url", c.GitLabURL)
-			if c.Name != "" {
-				cmd.Args = append(cmd.Args, "-name", c.Name)
-			}
-			if c.Exclude != "" {
-				cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
-			}
-			if c.OnlyPublic {
-				cmd.Args = append(cmd.Args, "-public")
-			}
-			if c.CredentialPath != "" {
-				cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
-			}
-		} else if c.GerritApiURL != "" {
-			cmd = exec.Command("zoekt-mirror-gerrit",
-				"-dest", repoDir)
-			if c.CredentialPath != "" {
-				cmd.Args = append(cmd.Args, "-http-credentials", c.CredentialPath)
-			}
-			if c.Name != "" {
-				cmd.Args = append(cmd.Args, "-name", c.Name)
-			}
-			if c.Exclude != "" {
-				cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
-			}
-			cmd.Args = append(cmd.Args, c.GerritApiURL)
-		}
-
-		stdout, _ := loggedRun(cmd)
-
-		for _, fn := range bytes.Split(stdout, []byte{'\n'}) {
-			if len(fn) == 0 {
-				continue
-			}
-
-			pendingRepos <- string(fn)
-		}
-
-	}
-}
diff --git a/cmd/zoekt-indexserver/main.go b/cmd/zoekt-indexserver/main.go
deleted file mode 100644
index 8392bb8..0000000
--- a/cmd/zoekt-indexserver/main.go
+++ /dev/null
@@ -1,298 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This program manages a zoekt indexing deployment:
-// * recycling logs
-// * periodically fetching new data.
-// * periodically reindexing all git repos.
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"math"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"time"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/gitindex"
-)
-
-const day = time.Hour * 24
-
-func loggedRun(cmd *exec.Cmd) (out, err []byte) {
-	outBuf := &bytes.Buffer{}
-	errBuf := &bytes.Buffer{}
-	cmd.Stdout = outBuf
-	cmd.Stderr = errBuf
-
-	log.Printf("run %v", cmd.Args)
-	if err := cmd.Run(); err != nil {
-		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
-			cmd.Args, err, outBuf.String(), errBuf.String())
-	}
-
-	return outBuf.Bytes(), errBuf.Bytes()
-}
-
-type Options struct {
-	cpuFraction      float64
-	cpuCount         int
-	fetchInterval    time.Duration
-	mirrorInterval   time.Duration
-	indexFlagsStr    string
-	indexFlags       []string
-	mirrorConfigFile string
-	maxLogAge        time.Duration
-	indexTimeout     time.Duration
-}
-
-func (o *Options) validate() {
-	if o.cpuFraction <= 0.0 || o.cpuFraction > 1.0 {
-		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
-	}
-
-	o.cpuCount = int(math.Trunc(float64(runtime.GOMAXPROCS(0)) * o.cpuFraction))
-	if o.cpuCount < 1 {
-		o.cpuCount = 1
-	}
-	if o.indexFlagsStr != "" {
-		o.indexFlags = strings.Split(o.indexFlagsStr, " ")
-	}
-}
-
-func (o *Options) defineFlags() {
-	flag.DurationVar(&o.indexTimeout, "index_timeout", time.Hour, "kill index job after this much time")
-	flag.DurationVar(&o.maxLogAge, "max_log_age", 3*day, "recycle index logs after this much time")
-	flag.DurationVar(&o.fetchInterval, "fetch_interval", time.Hour, "run fetches this often")
-	flag.StringVar(&o.mirrorConfigFile, "mirror_config",
-		"", "JSON file holding mirror configuration.")
-
-	flag.DurationVar(&o.mirrorInterval, "mirror_duration", 24*time.Hour, "find and clone new repos at this frequency.")
-	flag.Float64Var(&o.cpuFraction, "cpu_fraction", 0.25,
-		"use this fraction of the cores for indexing.")
-	flag.StringVar(&o.indexFlagsStr, "git_index_flags", "", "space separated list of flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false -submodules=false'")
-}
-
-// periodicFetch runs git-fetch every once in a while. Results are
-// posted on pendingRepos.
-func periodicFetch(repoDir, indexDir string, opts *Options, pendingRepos chan<- string) {
-	t := time.NewTicker(opts.fetchInterval)
-	for {
-		repos, err := gitindex.FindGitRepos(repoDir)
-		if err != nil {
-			log.Println(err)
-			continue
-		}
-		if len(repos) == 0 {
-			log.Printf("no repos found under %s", repoDir)
-		}
-
-		// TODO: Randomize to make sure quota throttling hits everyone.
-
-		later := map[string]struct{}{}
-		for _, dir := range repos {
-			if ok := fetchGitRepo(dir); !ok {
-				later[dir] = struct{}{}
-			} else {
-				pendingRepos <- dir
-			}
-		}
-
-		for r := range later {
-			pendingRepos <- r
-		}
-
-		<-t.C
-	}
-}
-
-// fetchGitRepo runs git-fetch, and returns true if there was an
-// update.
-func fetchGitRepo(dir string) bool {
-	cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
-	outBuf := &bytes.Buffer{}
-	errBuf := &bytes.Buffer{}
-
-	// Prevent prompting
-	cmd.Stdin = &bytes.Buffer{}
-	cmd.Stderr = errBuf
-	cmd.Stdout = outBuf
-	if err := cmd.Run(); err != nil {
-		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
-			cmd.Args, err, outBuf.String(), errBuf.String())
-	} else {
-		return len(outBuf.Bytes()) != 0
-	}
-	return false
-}
-
-// indexPendingRepos consumes the directories on the repos channel and
-// indexes them, sequentially.
-func indexPendingRepos(indexDir, repoDir string, opts *Options, repos <-chan string) {
-	for dir := range repos {
-		indexPendingRepo(dir, indexDir, repoDir, opts)
-
-		// Failures (eg. timeout) will leave temp files
-		// around. We have to clean them, or they will fill up the indexing volume.
-		if failures, err := filepath.Glob(filepath.Join(indexDir, "*.tmp")); err != nil {
-			log.Printf("Glob: %v", err)
-		} else {
-			for _, f := range failures {
-				os.Remove(f)
-			}
-		}
-	}
-}
-
-func indexPendingRepo(dir, indexDir, repoDir string, opts *Options) {
-	ctx, cancel := context.WithTimeout(context.Background(), opts.indexTimeout)
-	defer cancel()
-	args := []string{
-		"-require_ctags",
-		fmt.Sprintf("-parallelism=%d", opts.cpuCount),
-		"-repo_cache", repoDir,
-		"-index", indexDir,
-		"-incremental",
-	}
-	args = append(args, opts.indexFlags...)
-	args = append(args, dir)
-	cmd := exec.CommandContext(ctx, "zoekt-git-index", args...)
-	loggedRun(cmd)
-}
-
-// deleteLogs deletes old logs.
-func deleteLogs(logDir string, maxAge time.Duration) {
-	fs, err := filepath.Glob(filepath.Join(logDir, "*"))
-	if err != nil {
-		log.Fatalf("filepath.Glob(%s): %v", logDir, err)
-	}
-
-	threshold := time.Now().Add(-maxAge)
-	for _, fn := range fs {
-		if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {
-			os.Remove(fn)
-		}
-	}
-}
-
-func deleteLogsLoop(logDir string, maxAge time.Duration) {
-	tick := time.NewTicker(maxAge / 100)
-	for {
-		deleteLogs(logDir, maxAge)
-		<-tick.C
-	}
-}
-
-// Delete the shard if its corresponding git repo can't be found.
-func deleteIfOrphan(repoDir string, fn string) error {
-	f, err := os.Open(fn)
-	if err != nil {
-		return nil
-	}
-	defer f.Close()
-
-	ifile, err := zoekt.NewIndexFile(f)
-	if err != nil {
-		return nil
-	}
-	defer ifile.Close()
-
-	repo, _, err := zoekt.ReadMetadata(ifile)
-	if err != nil {
-		return nil
-	}
-
-	_, err = os.Stat(repo.Source)
-	if os.IsNotExist(err) {
-		log.Printf("deleting orphan shard %s; source %q not found", fn, repo.Source)
-		return os.Remove(fn)
-	}
-
-	return err
-}
-
-func deleteOrphanIndexes(indexDir, repoDir string, watchInterval time.Duration) {
-	t := time.NewTicker(watchInterval)
-
-	expr := indexDir + "/*"
-	for {
-		fs, err := filepath.Glob(expr)
-		if err != nil {
-			log.Printf("Glob(%q): %v", expr, err)
-		}
-
-		for _, f := range fs {
-			if err := deleteIfOrphan(repoDir, f); err != nil {
-				log.Printf("deleteIfOrphan(%q): %v", f, err)
-			}
-		}
-		<-t.C
-	}
-}
-
-func main() {
-	var opts Options
-	opts.defineFlags()
-	dataDir := flag.String("data_dir",
-		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
-	indexDir := flag.String("index_dir", "", "directory holding index shards. Defaults to $data_dir/index/")
-	flag.Parse()
-	opts.validate()
-
-	if *dataDir == "" {
-		log.Fatal("must set --data_dir")
-	}
-
-	// Automatically prepend our own path at the front, to minimize
-	// required configuration.
-	if l, err := os.Readlink("/proc/self/exe"); err == nil {
-		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
-	}
-
-	logDir := filepath.Join(*dataDir, "logs")
-	if *indexDir == "" {
-		*indexDir = filepath.Join(*dataDir, "index")
-	}
-	repoDir := filepath.Join(*dataDir, "repos")
-	for _, s := range []string{logDir, *indexDir, repoDir} {
-		if _, err := os.Stat(s); err == nil {
-			continue
-		}
-
-		if err := os.MkdirAll(s, 0o755); err != nil {
-			log.Fatalf("MkdirAll %s: %v", s, err)
-		}
-	}
-
-	_, err := readConfigURL(opts.mirrorConfigFile)
-	if err != nil {
-		log.Fatalf("readConfigURL(%s): %v", opts.mirrorConfigFile, err)
-	}
-
-	pendingRepos := make(chan string, 10)
-	go periodicMirrorFile(repoDir, &opts, pendingRepos)
-	go deleteLogsLoop(logDir, opts.maxLogAge)
-	go deleteOrphanIndexes(*indexDir, repoDir, opts.fetchInterval)
-	go indexPendingRepos(*indexDir, repoDir, &opts, pendingRepos)
-	periodicFetch(repoDir, *indexDir, &opts, pendingRepos)
-}
diff --git a/cmd/zoekt-mirror-bitbucket-server/main.go b/cmd/zoekt-mirror-bitbucket-server/main.go
deleted file mode 100644
index 9cfb204..0000000
--- a/cmd/zoekt-mirror-bitbucket-server/main.go
+++ /dev/null
@@ -1,270 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary fetches all repos of a project, and of a specific type, in case
-// these are specified, and clones them. By default it fetches and clones all
-// existing repos.
-package main
-
-import (
-	"context"
-	"crypto/tls"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/gfleury/go-bitbucket-v1"
-
-	"github.com/google/zoekt/gitindex"
-)
-
-func main() {
-	dest := flag.String("dest", "", "destination directory")
-	serverUrl := flag.String("url", "", "BitBucket Server url")
-	disableTLS := flag.Bool("disable-tls", false, "disables TLS verification")
-	credentialsFile := flag.String("credentials", ".bitbucket-credentials", "file holding BitBucket Server credentials")
-	project := flag.String("project", "", "project to mirror")
-	deleteRepos := flag.Bool("delete", false, "delete missing repos")
-	namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
-	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
-	projectType := flag.String("type", "", "only clone repos whose type matches the given string. "+
-		"Type can be either NORMAl or PERSONAL. Clones projects of both types if not set.")
-	flag.Parse()
-
-	if *serverUrl == "" {
-		log.Fatal("must set --url")
-	}
-
-	rootURL, err := url.Parse(*serverUrl)
-	if err != nil {
-		log.Fatalf("url.Parse(): %v", err)
-	}
-
-	if *dest == "" {
-		log.Fatal("must set --dest")
-	}
-
-	if *projectType != "" && !IsValidProjectType(*projectType) {
-		log.Fatal("type should be either NORMAL or PERSONAL")
-	}
-
-	destDir := filepath.Join(*dest, rootURL.Host)
-	if err := os.MkdirAll(destDir, 0o755); err != nil {
-		log.Fatal(err)
-	}
-
-	username := ""
-	password := ""
-	if *credentialsFile == "" {
-		log.Fatal("must set --credentials")
-	} else {
-		content, err := ioutil.ReadFile(*credentialsFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		credentials := strings.Fields(string(content))
-		username, password = credentials[0], credentials[1]
-	}
-
-	basicAuth := bitbucketv1.BasicAuth{UserName: username, Password: password}
-	ctx, cancel := context.WithTimeout(context.Background(), 120000*time.Millisecond)
-	ctx = context.WithValue(ctx, bitbucketv1.ContextBasicAuth, basicAuth)
-	defer cancel()
-
-	apiPath, err := url.Parse("/rest")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	apiBaseURL := rootURL.ResolveReference(apiPath).String()
-
-	var config *bitbucketv1.Configuration
-	if *disableTLS {
-		tr := &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		}
-		httpClient := &http.Client{
-			Transport: tr,
-		}
-		httpClientConfig := func(configs *bitbucketv1.Configuration) {
-			configs.HTTPClient = httpClient
-		}
-		config = bitbucketv1.NewConfiguration(apiBaseURL, httpClientConfig)
-	} else {
-		config = bitbucketv1.NewConfiguration(apiBaseURL)
-	}
-	client := bitbucketv1.NewAPIClient(ctx, config)
-
-	var repos []bitbucketv1.Repository
-
-	if *project != "" {
-		repos, err = getProjectRepos(*client, *project)
-	} else {
-		repos, err = getAllRepos(*client)
-	}
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	trimmed := repos[:0]
-	for _, r := range repos {
-		if filter.Include(r.Slug) && (*projectType == "" || r.Project.Type == *projectType) {
-			trimmed = append(trimmed, r)
-		}
-	}
-	repos = trimmed
-
-	if err := cloneRepos(destDir, rootURL.Host, repos, password); err != nil {
-		log.Fatalf("cloneRepos: %v", err)
-	}
-
-	if *deleteRepos {
-		if err := deleteStaleRepos(*dest, filter, repos); err != nil {
-			log.Fatalf("deleteStaleRepos: %v", err)
-		}
-	}
-}
-
-func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []bitbucketv1.Repository) error {
-	var baseURL string
-	if len(repos) > 0 {
-		baseURL = repos[0].Links.Self[0].Href
-	} else {
-		return nil
-	}
-	u, err := url.Parse(baseURL)
-	if err != nil {
-		return err
-	}
-	u.Path = ""
-
-	names := map[string]struct{}{}
-	for _, r := range repos {
-		names[filepath.Join(u.Host, r.Project.Key, r.Slug+".git")] = struct{}{}
-	}
-
-	if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
-		log.Fatalf("deleteRepos: %v", err)
-	}
-	return nil
-}
-
-func IsValidProjectType(projectType string) bool {
-	switch projectType {
-	case "NORMAL", "PERSONAL":
-		return true
-	}
-	return false
-}
-
-func getAllRepos(client bitbucketv1.APIClient) ([]bitbucketv1.Repository, error) {
-	var allRepos []bitbucketv1.Repository
-	opts := map[string]interface{}{
-		"limit": 1000,
-		"start": 0,
-	}
-
-	for {
-		resp, err := client.DefaultApi.GetRepositories_19(opts)
-		if err != nil {
-			return nil, err
-		}
-
-		repos, err := bitbucketv1.GetRepositoriesResponse(resp)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(repos) == 0 {
-			break
-		}
-
-		opts["start"] = opts["start"].(int) + opts["limit"].(int)
-
-		allRepos = append(allRepos, repos...)
-	}
-	return allRepos, nil
-}
-
-func getProjectRepos(client bitbucketv1.APIClient, projectName string) ([]bitbucketv1.Repository, error) {
-	var allRepos []bitbucketv1.Repository
-	opts := map[string]interface{}{
-		"limit": 1000,
-		"start": 0,
-	}
-
-	for {
-		resp, err := client.DefaultApi.GetRepositoriesWithOptions(projectName, opts)
-		if err != nil {
-			return nil, err
-		}
-
-		repos, err := bitbucketv1.GetRepositoriesResponse(resp)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(repos) == 0 {
-			break
-		}
-
-		opts["start"] = opts["start"].(int) + opts["limit"].(int)
-
-		allRepos = append(allRepos, repos...)
-	}
-	return allRepos, nil
-}
-
-func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, password string) error {
-	for _, r := range repos {
-		fullName := filepath.Join(r.Project.Key, r.Slug)
-		config := map[string]string{
-			"zoekt.web-url-type": "bitbucket-server",
-			"zoekt.web-url":      r.Links.Self[0].Href,
-			"zoekt.name":         filepath.Join(host, fullName),
-		}
-
-		httpsCloneUrl := ""
-		for _, cloneUrl := range r.Links.Clone {
-			// In fact, this is an https url, i.e. there's no separate Name for https.
-			if cloneUrl.Name == "http" {
-				s := strings.Split(cloneUrl.Href, "@")
-				httpsCloneUrl = s[0] + ":" + password + "@" + s[1]
-			}
-		}
-
-		if httpsCloneUrl != "" {
-			dest, err := gitindex.CloneRepo(destDir, fullName, httpsCloneUrl, config)
-			if err != nil {
-				return err
-			}
-			if dest != "" {
-				fmt.Println(dest)
-			}
-		}
-	}
-
-	return nil
-}
diff --git a/cmd/zoekt-mirror-gerrit/main.go b/cmd/zoekt-mirror-gerrit/main.go
deleted file mode 100644
index b186de7..0000000
--- a/cmd/zoekt-mirror-gerrit/main.go
+++ /dev/null
@@ -1,182 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary fetches all repos of a Gerrit host.
-
-package main
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"path/filepath"
-	"strconv"
-	"strings"
-
-	gerrit "github.com/andygrunwald/go-gerrit"
-	"github.com/google/zoekt/gitindex"
-)
-
-type loggingRT struct {
-	http.RoundTripper
-}
-
-type closeBuffer struct {
-	*bytes.Buffer
-}
-
-func (b *closeBuffer) Close() error { return nil }
-
-const debug = false
-
-func (rt *loggingRT) RoundTrip(req *http.Request) (rep *http.Response, err error) {
-	if debug {
-		log.Println("Req: ", req)
-	}
-	rep, err = rt.RoundTripper.RoundTrip(req)
-	if debug {
-		log.Println("Rep: ", rep, err)
-	}
-	if err == nil {
-		body, _ := ioutil.ReadAll(rep.Body)
-
-		rep.Body.Close()
-		if debug {
-			log.Println("body: ", string(body))
-		}
-		rep.Body = &closeBuffer{bytes.NewBuffer(body)}
-	}
-	return rep, err
-}
-
-func newLoggingClient() *http.Client {
-	return &http.Client{
-		Transport: &loggingRT{
-			RoundTripper: http.DefaultTransport,
-		},
-	}
-}
-
-func main() {
-	dest := flag.String("dest", "", "destination directory")
-	namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
-	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
-	httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
-	flag.Parse()
-
-	if len(flag.Args()) < 1 {
-		log.Fatal("must provide URL argument.")
-	}
-
-	rootURL, err := url.Parse(flag.Arg(0))
-	if err != nil {
-		log.Fatalf("url.Parse(): %v", err)
-	}
-
-	if *httpCrendentialsPath != "" {
-		creds, err := ioutil.ReadFile(*httpCrendentialsPath)
-		if err != nil {
-			log.Print("Cannot read gerrit http credentials, going Anonymous")
-		} else {
-			splitCreds := strings.Split(strings.TrimSpace(string(creds)), ":")
-			rootURL.User = url.UserPassword(splitCreds[0], splitCreds[1])
-		}
-	}
-
-	if *dest == "" {
-		log.Fatal("must set --dest")
-	}
-
-	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	client, err := gerrit.NewClient(rootURL.String(), newLoggingClient())
-	if err != nil {
-		log.Fatalf("NewClient(%s): %v", rootURL, err)
-	}
-
-	info, _, err := client.Config.GetServerInfo()
-	if err != nil {
-		log.Fatalf("GetServerInfo: %v", err)
-	}
-
-	var projectURL string
-	for _, s := range []string{"http", "anonymous http"} {
-		projectURL = info.Download.Schemes[s].URL
-	}
-	if projectURL == "" {
-		log.Fatalf("project URL is empty, got Schemes %#v", info.Download.Schemes)
-	}
-
-	projects := make(map[string]gerrit.ProjectInfo)
-	skip := "0"
-	for {
-		page, _, err := client.Projects.ListProjects(&gerrit.ProjectOptions{Skip: skip})
-		if err != nil {
-			log.Fatalf("ListProjects: %v", err)
-		}
-
-		if len(*page) == 0 {
-			break
-		}
-		for k, v := range *page {
-			projects[k] = v
-		}
-		skip = strconv.Itoa(len(projects))
-	}
-
-	for k, v := range projects {
-		if !filter.Include(k) {
-			continue
-		}
-
-		cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, -1))
-		if err != nil {
-			log.Fatalf("url.Parse: %v", err)
-		}
-
-		name := filepath.Join(cloneURL.Host, cloneURL.Path)
-		config := map[string]string{
-			"zoekt.name":           name,
-			"zoekt.gerrit-project": k,
-			"zoekt.gerrit-host":    rootURL.String(),
-		}
-
-		for _, wl := range v.WebLinks {
-			// default gerrit gitiles config is named browse, and does not include
-			// root domain name in it. Cheating.
-			switch wl.Name {
-			case "browse":
-				config["zoekt.web-url"] = fmt.Sprintf("%s://%s%s", rootURL.Scheme,
-					rootURL.Host, wl.URL)
-				config["zoekt.web-url-type"] = "gitiles"
-			default:
-				config["zoekt.web-url"] = wl.URL
-				config["zoekt.web-url-type"] = wl.Name
-			}
-		}
-
-		if dest, err := gitindex.CloneRepo(*dest, name, cloneURL.String(), config); err != nil {
-			log.Fatalf("CloneRepo: %v", err)
-		} else {
-			fmt.Println(dest)
-		}
-	}
-}
diff --git a/cmd/zoekt-mirror-github/main.go b/cmd/zoekt-mirror-github/main.go
deleted file mode 100644
index 1b0d8f8..0000000
--- a/cmd/zoekt-mirror-github/main.go
+++ /dev/null
@@ -1,314 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary fetches all repos of a user or organization and clones
-// them.  It is strongly recommended to get a personal API token from
-// https://github.com/settings/tokens, save the token in a file, and
-// point the --token option to it.
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-
-	"github.com/google/go-github/v27/github"
-	"golang.org/x/oauth2"
-
-	"github.com/google/zoekt/gitindex"
-)
-
-type topicsFlag []string
-
-func (f *topicsFlag) String() string {
-	return strings.Join(*f, ",")
-}
-
-func (f *topicsFlag) Set(value string) error {
-	*f = append(*f, value)
-	return nil
-}
-
-type reposFilters struct {
-	topics        []string
-	excludeTopics []string
-}
-
-func main() {
-	dest := flag.String("dest", "", "destination directory")
-	githubURL := flag.String("url", "", "GitHub Enterprise url. If not set github.com will be used as the host.")
-	org := flag.String("org", "", "organization to mirror")
-	user := flag.String("user", "", "user to mirror")
-	token := flag.String("token",
-		filepath.Join(os.Getenv("HOME"), ".github-token"),
-		"file holding API token.")
-	forks := flag.Bool("forks", false, "also mirror forks.")
-	deleteRepos := flag.Bool("delete", false, "delete missing repos")
-	namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
-	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
-	topics := topicsFlag{}
-	flag.Var(&topics, "topic", "only clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
-	excludeTopics := topicsFlag{}
-	flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
-
-	flag.Parse()
-
-	if *dest == "" {
-		log.Fatal("must set --dest")
-	}
-	if *githubURL == "" && *org == "" && *user == "" {
-		log.Fatal("must set either --org or --user when github.com is used as host")
-	}
-
-	var host string
-	var apiBaseURL string
-	var client *github.Client
-	if *githubURL != "" {
-		rootURL, err := url.Parse(*githubURL)
-		if err != nil {
-			log.Fatal(err)
-		}
-		host = rootURL.Host
-		apiPath, err := url.Parse("/api/v3/")
-		if err != nil {
-			log.Fatal(err)
-		}
-		apiBaseURL = rootURL.ResolveReference(apiPath).String()
-		client, err = github.NewEnterpriseClient(apiBaseURL, apiBaseURL, nil)
-		if err != nil {
-			log.Fatal(err)
-		}
-	} else {
-		host = "github.com"
-		apiBaseURL = "https://github.com/"
-		client = github.NewClient(nil)
-	}
-	destDir := filepath.Join(*dest, host)
-	if err := os.MkdirAll(destDir, 0o755); err != nil {
-		log.Fatal(err)
-	}
-
-	if *token != "" {
-		content, err := ioutil.ReadFile(*token)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		ts := oauth2.StaticTokenSource(
-			&oauth2.Token{
-				AccessToken: strings.TrimSpace(string(content)),
-			})
-		tc := oauth2.NewClient(context.Background(), ts)
-		if *githubURL != "" {
-			client, err = github.NewEnterpriseClient(apiBaseURL, apiBaseURL, tc)
-			if err != nil {
-				log.Fatal(err)
-			}
-		} else {
-			client = github.NewClient(tc)
-		}
-	}
-
-	reposFilters := reposFilters{
-		topics:        topics,
-		excludeTopics: excludeTopics,
-	}
-	var repos []*github.Repository
-	var err error
-	if *org != "" {
-		repos, err = getOrgRepos(client, *org, reposFilters)
-	} else if *user != "" {
-		repos, err = getUserRepos(client, *user, reposFilters)
-	} else {
-		log.Printf("no user or org specified, cloning all repos.")
-		repos, err = getUserRepos(client, "", reposFilters)
-	}
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if !*forks {
-		trimmed := repos[:0]
-		for _, r := range repos {
-			if r.Fork == nil || !*r.Fork {
-				trimmed = append(trimmed, r)
-			}
-		}
-		repos = trimmed
-	}
-
-	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	{
-		trimmed := repos[:0]
-		for _, r := range repos {
-			if filter.Include(*r.Name) {
-				trimmed = append(trimmed, r)
-			}
-		}
-		repos = trimmed
-	}
-
-	if err := cloneRepos(destDir, repos); err != nil {
-		log.Fatalf("cloneRepos: %v", err)
-	}
-
-	if *deleteRepos {
-		if err := deleteStaleRepos(*dest, filter, repos, *org+*user); err != nil {
-			log.Fatalf("deleteStaleRepos: %v", err)
-		}
-	}
-}
-
-func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []*github.Repository, user string) error {
-	var baseURL string
-	if len(repos) > 0 {
-		baseURL = *repos[0].HTMLURL
-	} else {
-		return nil
-	}
-	u, err := url.Parse(baseURL)
-	if err != nil {
-		return err
-	}
-	u.Path = user
-
-	names := map[string]struct{}{}
-	for _, r := range repos {
-		u, err := url.Parse(*r.HTMLURL)
-		if err != nil {
-			return err
-		}
-
-		names[filepath.Join(u.Host, u.Path+".git")] = struct{}{}
-	}
-	if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
-		log.Fatalf("deleteRepos: %v", err)
-	}
-	return nil
-}
-
-func hasIntersection(s1, s2 []string) bool {
-	hash := make(map[string]bool)
-	for _, e := range s1 {
-		hash[e] = true
-	}
-	for _, e := range s2 {
-		if hash[e] {
-			return true
-		}
-	}
-	return false
-}
-
-func filterByTopic(repos []*github.Repository, include []string, exclude []string) (filteredRepos []*github.Repository) {
-	for _, repo := range repos {
-		if (len(include) == 0 || hasIntersection(include, repo.Topics)) &&
-			!hasIntersection(exclude, repo.Topics) {
-			filteredRepos = append(filteredRepos, repo)
-		}
-	}
-	return
-}
-
-func getOrgRepos(client *github.Client, org string, reposFilters reposFilters) ([]*github.Repository, error) {
-	var allRepos []*github.Repository
-	opt := &github.RepositoryListByOrgOptions{}
-	for {
-		repos, resp, err := client.Repositories.ListByOrg(context.Background(), org, opt)
-		if err != nil {
-			return nil, err
-		}
-		if len(repos) == 0 {
-			break
-		}
-
-		opt.Page = resp.NextPage
-		repos = filterByTopic(repos, reposFilters.topics, reposFilters.excludeTopics)
-		allRepos = append(allRepos, repos...)
-		if resp.NextPage == 0 {
-			break
-		}
-	}
-	return allRepos, nil
-}
-
-func getUserRepos(client *github.Client, user string, reposFilters reposFilters) ([]*github.Repository, error) {
-	var allRepos []*github.Repository
-	opt := &github.RepositoryListOptions{}
-	for {
-		repos, resp, err := client.Repositories.List(context.Background(), user, opt)
-		if err != nil {
-			return nil, err
-		}
-		if len(repos) == 0 {
-			break
-		}
-
-		opt.Page = resp.NextPage
-		repos = filterByTopic(repos, reposFilters.topics, reposFilters.excludeTopics)
-		allRepos = append(allRepos, repos...)
-		if resp.NextPage == 0 {
-			break
-		}
-	}
-	return allRepos, nil
-}
-
-func itoa(p *int) string {
-	if p != nil {
-		return strconv.Itoa(*p)
-	}
-	return ""
-}
-
-func cloneRepos(destDir string, repos []*github.Repository) error {
-	for _, r := range repos {
-		host, err := url.Parse(*r.HTMLURL)
-		if err != nil {
-			return err
-		}
-		config := map[string]string{
-			"zoekt.web-url-type": "github",
-			"zoekt.web-url":      *r.HTMLURL,
-			"zoekt.name":         filepath.Join(host.Hostname(), *r.FullName),
-
-			"zoekt.github-stars":       itoa(r.StargazersCount),
-			"zoekt.github-watchers":    itoa(r.WatchersCount),
-			"zoekt.github-subscribers": itoa(r.SubscribersCount),
-			"zoekt.github-forks":       itoa(r.ForksCount),
-		}
-		dest, err := gitindex.CloneRepo(destDir, *r.FullName, *r.CloneURL, config)
-		if err != nil {
-			return err
-		}
-		if dest != "" {
-			fmt.Println(dest)
-		}
-
-	}
-
-	return nil
-}
diff --git a/cmd/zoekt-mirror-gitiles/cgit.go b/cmd/zoekt-mirror-gitiles/cgit.go
deleted file mode 100644
index 7517cf8..0000000
--- a/cmd/zoekt-mirror-gitiles/cgit.go
+++ /dev/null
@@ -1,118 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
-)
-
-// I will go to programmer hell for trying to parse HTML with
-// regexps. Why doesn't CGit have a JSON interface?
-var cgitRepoEntryRE = regexp.MustCompile(
-	`class='sublevel-repo'><a title='([^'"]*)' href='([^']*)'>`)
-
-func normalizedGet(u *url.URL) ([]byte, error) {
-	rep, err := http.Get(u.String())
-	if err != nil {
-		return nil, err
-	}
-	defer rep.Body.Close()
-	if rep.StatusCode != 200 {
-		return nil, fmt.Errorf("status %s", rep.Status)
-	}
-
-	c, err := ioutil.ReadAll(rep.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	c = bytes.Replace(c, []byte{'\n'}, []byte{' '}, -1)
-	return c, nil
-}
-
-// getCGitRepos finds repo names from the CGit index page hosted at
-// URL `u`.
-func getCGitRepos(u *url.URL, filter func(string) bool) (map[string]*crawlTarget, error) {
-	c, err := normalizedGet(u)
-	if err != nil {
-		return nil, err
-	}
-
-	pages := map[string]*crawlTarget{}
-	for _, m := range cgitRepoEntryRE.FindAllSubmatch(c, -1) {
-		nm := strings.TrimSuffix(string(m[1]), ".git")
-
-		if !filter(nm) {
-			continue
-		}
-
-		relUrl := string(m[2])
-
-		u, err := u.Parse(relUrl)
-		if err != nil {
-			log.Printf("ignoring u.Parse(%q): %v", relUrl, err)
-			continue
-		}
-		pages[nm] = &crawlTarget{
-			webURL:     u.String(),
-			webURLType: "cgit",
-		}
-	}
-
-	// TODO - parallel?
-	for _, target := range pages {
-		u, _ := url.Parse(target.webURL)
-		c, err := cgitCloneURL(u)
-		if err != nil {
-			log.Printf("ignoring cgitCloneURL(%s): %v", u, c)
-			continue
-		}
-
-		target.cloneURL = c.String()
-	}
-	return pages, nil
-}
-
-// We'll take the first URL we get. This may put the git:// URL (which
-// is insecure) at the top, but individual machines (such as
-// git.savannah.gnu) probably would rather receive git:// traffic
-// which is more efficient.
-
-// TODO - do something like `Clone.*<a.*href=` to get the first
-// URL. Older versions don't say vcs-git.
-var cloneURLRe = regexp.MustCompile(
-	`rel=["']vcs-git["'] *href=["']([^"']*)["']`)
-
-func cgitCloneURL(u *url.URL) (*url.URL, error) {
-	c, err := normalizedGet(u)
-	if err != nil {
-		return nil, err
-	}
-
-	m := cloneURLRe.FindSubmatch(c)
-	cl, err := url.Parse(string(m[1]))
-	if err != nil {
-		return nil, err
-	}
-
-	return cl, nil
-}
diff --git a/cmd/zoekt-mirror-gitiles/gitiles.go b/cmd/zoekt-mirror-gitiles/gitiles.go
deleted file mode 100644
index 9510d10..0000000
--- a/cmd/zoekt-mirror-gitiles/gitiles.go
+++ /dev/null
@@ -1,70 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"path"
-)
-
-type Project struct {
-	Name     string
-	CloneURL string `json:"clone_url"`
-}
-
-func getGitilesRepos(root *url.URL, filter func(string) bool) (map[string]*crawlTarget, error) {
-	jsRoot := *root
-	jsRoot.RawQuery = "format=JSON"
-	resp, err := http.Get(jsRoot.String())
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	content, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	const xssTag = ")]}'\n"
-	content = bytes.TrimPrefix(content, []byte(xssTag))
-
-	m := map[string]*Project{}
-	if err := json.Unmarshal(content, &m); err != nil {
-		return nil, err
-	}
-
-	result := map[string]*crawlTarget{}
-	for k, v := range m {
-		if k == "All-Users" || k == "All-Projects" {
-			continue
-		}
-		if !filter(k) {
-			continue
-		}
-		web := *root
-		web.Path = path.Join(web.Path, v.Name)
-		result[k] = &crawlTarget{
-			cloneURL:   v.CloneURL,
-			webURL:     web.String(),
-			webURLType: "gitiles",
-		}
-	}
-	return result, nil
-}
diff --git a/cmd/zoekt-mirror-gitiles/main.go b/cmd/zoekt-mirror-gitiles/main.go
deleted file mode 100644
index c4210a9..0000000
--- a/cmd/zoekt-mirror-gitiles/main.go
+++ /dev/null
@@ -1,102 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary fetches all repos of a Gitiles host.  It does double
-// duty for other "simple" web hosts
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"net/url"
-	"os"
-	"path/filepath"
-
-	"github.com/google/zoekt/gitindex"
-)
-
-type crawlTarget struct {
-	cloneURL   string
-	webURL     string
-	webURLType string
-}
-
-type hostCrawler func(*url.URL, func(string) bool) (map[string]*crawlTarget, error)
-
-func main() {
-	dest := flag.String("dest", "", "destination directory")
-	namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
-	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
-	hostType := flag.String("type", "gitiles", "which webserver to crawl. Choices: gitiles, cgit")
-	flag.Parse()
-
-	if len(flag.Args()) < 1 {
-		log.Fatal("must provide URL argument.")
-	}
-
-	var crawler hostCrawler
-	switch *hostType {
-	case "gitiles":
-		crawler = getGitilesRepos
-	case "cgit":
-		crawler = getCGitRepos
-	default:
-		log.Fatalf("unknown host type %q", *hostType)
-	}
-
-	rootURL, err := url.Parse(flag.Arg(0))
-	if err != nil {
-		log.Fatalf("url.Parse(): %v", err)
-	}
-
-	if *dest == "" {
-		log.Fatal("must set --dest")
-	}
-
-	if err := os.MkdirAll(filepath.Join(*dest, rootURL.Host, rootURL.Path), 0o755); err != nil {
-		log.Fatal(err)
-	}
-
-	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	repos, err := crawler(rootURL, filter.Include)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for nm, target := range repos {
-		// For git.savannah.gnu.org, this puts an ugly "CGit"
-		// path component into the name. However, it's
-		// possible that there are multiple, different CGit pages
-		// on the host, so we have to keep it.
-		fullName := filepath.Join(rootURL.Host, rootURL.Path, nm)
-		config := map[string]string{
-			"zoekt.web-url":      target.webURL,
-			"zoekt.web-url-type": target.webURLType,
-			"zoekt.name":         fullName,
-		}
-
-		dest, err := gitindex.CloneRepo(*dest, fullName, target.cloneURL, config)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if dest != "" {
-			fmt.Println(dest)
-		}
-	}
-}
diff --git a/cmd/zoekt-mirror-gitlab/main.go b/cmd/zoekt-mirror-gitlab/main.go
deleted file mode 100644
index e119e87..0000000
--- a/cmd/zoekt-mirror-gitlab/main.go
+++ /dev/null
@@ -1,189 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This binary fetches all repos for a user from gitlab.
-//
-// It is recommended to use a gitlab personal access token:
-// https://docs.gitlab.com/ce/user/profile/personal_access_tokens.html. This
-// token should be stored in a file and the --token option should be used.
-// In addition, the token should be present in the ~/.netrc of the user running
-// the mirror command. For example, the ~/.netrc may look like:
-//
-//   machine gitlab.com
-//   login oauth
-//   password <personal access token>
-//
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-
-	"github.com/google/zoekt/gitindex"
-	gitlab "github.com/xanzy/go-gitlab"
-)
-
-func main() {
-	dest := flag.String("dest", "", "destination directory")
-	gitlabURL := flag.String("url", "https://gitlab.com/api/v4/", "Gitlab URL. If not set https://gitlab.com/api/v4/ will be used")
-	token := flag.String("token",
-		filepath.Join(os.Getenv("HOME"), ".gitlab-token"),
-		"file holding API token.")
-	isMember := flag.Bool("membership", false, "only mirror repos this user is a member of ")
-	isPublic := flag.Bool("public", false, "only mirror public repos")
-	deleteRepos := flag.Bool("delete", false, "delete missing repos")
-	namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
-	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
-	flag.Parse()
-
-	if *dest == "" {
-		log.Fatal("must set --dest")
-	}
-
-	var host string
-	rootURL, err := url.Parse(*gitlabURL)
-	if err != nil {
-		log.Fatal(err)
-	}
-	host = rootURL.Host
-
-	destDir := filepath.Join(*dest, host)
-	if err := os.MkdirAll(destDir, 0o755); err != nil {
-		log.Fatal(err)
-	}
-
-	content, err := ioutil.ReadFile(*token)
-	if err != nil {
-		log.Fatal(err)
-	}
-	apiToken := strings.TrimSpace(string(content))
-
-	client := gitlab.NewClient(nil, apiToken)
-	client.SetBaseURL(*gitlabURL)
-
-	opt := &gitlab.ListProjectsOptions{
-		ListOptions: gitlab.ListOptions{
-			PerPage: 10,
-			Page:    1,
-		},
-		Membership: isMember,
-	}
-	if *isPublic {
-		opt.Visibility = gitlab.Visibility(gitlab.PublicVisibility)
-	}
-
-	var gitlabProjects []*gitlab.Project
-	for {
-		projects, resp, err := client.Projects.ListProjects(opt)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		for _, project := range projects {
-
-			// Skip projects without a default branch - these should be projects
-			// where the repository isn't enabled
-			if project.DefaultBranch == "" {
-				continue
-			}
-
-			gitlabProjects = append(gitlabProjects, project)
-		}
-
-		if resp.CurrentPage >= resp.TotalPages {
-			break
-		}
-
-		opt.Page = resp.NextPage
-	}
-
-	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	{
-		trimmed := gitlabProjects[:0]
-		for _, p := range gitlabProjects {
-			if filter.Include(p.NameWithNamespace) {
-				trimmed = append(trimmed, p)
-			}
-		}
-		gitlabProjects = trimmed
-	}
-
-	fetchProjects(destDir, apiToken, gitlabProjects)
-
-	if *deleteRepos {
-		if err := deleteStaleProjects(*dest, filter, gitlabProjects); err != nil {
-			log.Fatalf("deleteStaleProjects: %v", err)
-		}
-	}
-}
-
-func deleteStaleProjects(destDir string, filter *gitindex.Filter, projects []*gitlab.Project) error {
-	u, err := url.Parse(projects[0].HTTPURLToRepo)
-	u.Path = ""
-	if err != nil {
-		return err
-	}
-
-	names := map[string]struct{}{}
-	for _, p := range projects {
-		u, err := url.Parse(p.HTTPURLToRepo)
-		if err != nil {
-			return err
-		}
-
-		names[filepath.Join(u.Host, u.Path)] = struct{}{}
-	}
-
-	if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
-		log.Fatalf("deleteRepos: %v", err)
-	}
-	return nil
-}
-
-func fetchProjects(destDir, token string, projects []*gitlab.Project) {
-	for _, p := range projects {
-		u, err := url.Parse(p.HTTPURLToRepo)
-		if err != nil {
-			log.Printf("Unable to parse project URL: %v", err)
-			continue
-		}
-		config := map[string]string{
-			"zoekt.web-url-type": "gitlab",
-			"zoekt.web-url":      p.WebURL,
-			"zoekt.name":         filepath.Join(u.Hostname(), p.PathWithNamespace),
-
-			"zoekt.gitlab-stars": strconv.Itoa(p.StarCount),
-			"zoekt.gitlab-forks": strconv.Itoa(p.ForksCount),
-		}
-
-		cloneURL := p.HTTPURLToRepo
-		dest, err := gitindex.CloneRepo(destDir, p.PathWithNamespace, cloneURL, config)
-		if err != nil {
-			log.Printf("cloneRepos: %v", err)
-			continue
-		}
-		if dest != "" {
-			fmt.Println(dest)
-		}
-	}
-}
diff --git a/cmd/zoekt-repo-index/main.go b/cmd/zoekt-repo-index/main.go
deleted file mode 100644
index 06e8e20..0000000
--- a/cmd/zoekt-repo-index/main.go
+++ /dev/null
@@ -1,381 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/* zoekt-repo-index indexes a repo-based repository.  The constituent
-git repositories should already have been downloaded to the
---repo_cache directory, eg.
-
-    go install github.com/google/zoekt/cmd/zoekt-repo-index &&
-
-    zoekt-repo-index -base_url https://gfiber.googlesource.com/ \
-      -manifest_repo_url https://gfiber.googlesource.com/manifests \
-      -manifest_rev_prefix=refs/heads/ \
-      -rev_prefix="refs/remotes/" \
-      -repo_cache ~/zoekt-serving/repos/ \
-      -shard_limit 50000000 \
-       master:default_unrestricted.xml
-*/
-package main
-
-import (
-	"crypto/sha1"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/url"
-	"path"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	"github.com/google/slothfs/manifest"
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/gitindex"
-	"go.uber.org/automaxprocs/maxprocs"
-
-	git "github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-)
-
-var _ = log.Println
-
-type fileKey struct {
-	SubRepoPath string
-	Path        string
-	ID          plumbing.Hash
-}
-
-func (k *fileKey) FullPath() string {
-	return filepath.Join(k.SubRepoPath, k.Path)
-}
-
-type branchFile struct {
-	branch, file string
-	mf           *manifest.Manifest
-	manifestPath string
-}
-
-func parseBranches(manifestRepoURL, revPrefix string, cache *gitindex.RepoCache, args []string) ([]branchFile, error) {
-	var branches []branchFile
-	if manifestRepoURL != "" {
-		u, err := url.Parse(manifestRepoURL)
-		if err != nil {
-			return nil, err
-		}
-
-		repo, err := cache.Open(u)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, f := range args {
-			fs := strings.SplitN(f, ":", 2)
-			if len(fs) != 2 {
-				return nil, fmt.Errorf("cannot parse %q as BRANCH:FILE", f)
-			}
-			mf, err := getManifest(repo, revPrefix+fs[0], fs[1])
-			if err != nil {
-				return nil, fmt.Errorf("manifest %s:%s: %v", fs[0], fs[1], err)
-			}
-
-			branches = append(branches, branchFile{
-				branch:       fs[0],
-				file:         fs[1],
-				mf:           mf,
-				manifestPath: cache.Path(u),
-			})
-		}
-	} else {
-		if len(args) == 0 {
-			return nil, fmt.Errorf("must give XML file argument")
-		}
-		for _, f := range args {
-			mf, err := manifest.ParseFile(f)
-			if err != nil {
-				return nil, err
-			}
-
-			branches = append(branches, branchFile{
-				branch:       "HEAD",
-				file:         filepath.Base(f),
-				mf:           mf,
-				manifestPath: f,
-			})
-		}
-	}
-	return branches, nil
-}
-
-func main() {
-	sizeMax := flag.Int("file_limit", 128<<10, "maximum file size")
-	shardLimit := flag.Int("shard_limit", 100<<20, "maximum corpus size for a shard")
-	parallelism := flag.Int("parallelism", 1, "maximum number of parallel indexing processes")
-
-	revPrefix := flag.String("rev_prefix", "refs/remotes/origin/", "prefix for references")
-	baseURLStr := flag.String("base_url", "", "base url to interpret repository names")
-	repoCacheDir := flag.String("repo_cache", "", "root for repository cache")
-	indexDir := flag.String("index", build.DefaultDir, "index directory for *.zoekt files")
-	manifestRepoURL := flag.String("manifest_repo_url", "", "set a URL for a git repository holding manifest XML file. Provide the BRANCH:XML-FILE as further command-line arguments")
-	manifestRevPrefix := flag.String("manifest_rev_prefix", "refs/remotes/origin/", "prefixes for branches in manifest repository")
-	repoName := flag.String("name", "", "set repository name")
-	repoURL := flag.String("url", "", "set repository URL")
-	maxSubProjects := flag.Int("max_sub_projects", 0, "trim number of projects in manifest, for debugging.")
-	incremental := flag.Bool("incremental", true, "only index if the repository has changed.")
-	flag.Parse()
-
-	// Tune GOMAXPROCS to match Linux container CPU quota.
-	maxprocs.Set()
-
-	if *repoCacheDir == "" {
-		log.Fatal("must set --repo_cache")
-	}
-	repoCache := gitindex.NewRepoCache(*repoCacheDir)
-
-	if u, err := url.Parse(*baseURLStr); err != nil {
-		log.Fatalf("Parse(%q): %v", u, err)
-	} else if *repoName == "" {
-		*repoName = filepath.Join(u.Host, u.Path)
-	}
-
-	opts := build.Options{
-		Parallelism: *parallelism,
-		SizeMax:     *sizeMax,
-		ShardMax:    *shardLimit,
-		IndexDir:    *indexDir,
-		RepositoryDescription: zoekt.Repository{
-			Name: *repoName,
-			URL:  *repoURL,
-		},
-	}
-	opts.SetDefaults()
-	baseURL, err := url.Parse(*baseURLStr)
-	if err != nil {
-		log.Fatalf("Parse baseURL %q: %v", *baseURLStr, err)
-	}
-
-	branches, err := parseBranches(*manifestRepoURL, *manifestRevPrefix, repoCache, flag.Args())
-	if err != nil {
-		log.Fatalf("parseBranches(%s, %s): %v", *manifestRepoURL, *manifestRevPrefix, err)
-	}
-	if len(branches) == 0 {
-		log.Fatal("must specify at least one branch")
-	}
-	if *maxSubProjects > 0 {
-		for _, b := range branches {
-			if *maxSubProjects < len(b.mf.Project) {
-				b.mf.Project = b.mf.Project[:*maxSubProjects]
-			}
-		}
-	}
-
-	perBranch := map[string]map[fileKey]gitindex.BlobLocation{}
-	opts.SubRepositories = map[string]*zoekt.Repository{}
-
-	// branch => repo => version
-	versionMap := map[string]map[string]plumbing.Hash{}
-	for _, br := range branches {
-		br.mf.Filter()
-		files, versions, err := iterateManifest(br.mf, *baseURL, *revPrefix, repoCache)
-		if err != nil {
-			log.Fatalf("iterateManifest: %v", err)
-		}
-
-		perBranch[br.branch] = files
-		for key, loc := range files {
-			_, ok := opts.SubRepositories[key.SubRepoPath]
-			if ok {
-				// This can be incorrect: if the layout of manifests
-				// changes across branches, then the same file could
-				// be in different subRepos. We'll pretend this is not
-				// a problem.
-				continue
-			}
-
-			desc := &zoekt.Repository{}
-			if err := gitindex.SetTemplatesFromOrigin(desc, loc.URL); err != nil {
-				log.Fatalf("SetTemplatesFromOrigin(%s): %v", loc.URL, err)
-			}
-
-			opts.SubRepositories[key.SubRepoPath] = desc
-		}
-		versionMap[br.branch] = versions
-	}
-
-	for _, br := range branches {
-		var paths []string
-		for p := range opts.SubRepositories {
-			paths = append(paths, p)
-		}
-		sort.Strings(paths)
-
-		// Compute a version of the aggregate. This version
-		// has nothing to do with git, but will let us do
-		// incrementality correctly.
-		hasher := sha1.New()
-		for _, p := range paths {
-			repo := opts.SubRepositories[p]
-			id := versionMap[br.branch][p]
-
-			// it is possible that 'id' is zero, if this
-			// branch of the manifest doesn't have this
-			// particular subrepository.
-			hasher.Write([]byte(p))
-			hasher.Write([]byte(id.String()))
-			repo.Branches = append(repo.Branches, zoekt.RepositoryBranch{
-				Name:    br.branch,
-				Version: id.String(),
-			})
-		}
-
-		opts.RepositoryDescription.Branches = append(opts.RepositoryDescription.Branches, zoekt.RepositoryBranch{
-			Name:    br.branch,
-			Version: fmt.Sprintf("%x", hasher.Sum(nil)),
-		})
-	}
-
-	// key => branch
-	all := map[fileKey][]string{}
-	for br, files := range perBranch {
-		for k := range files {
-			all[k] = append(all[k], br)
-		}
-	}
-
-	if *incremental && opts.IncrementalSkipIndexing() {
-		return
-	}
-
-	builder, err := build.NewBuilder(opts)
-	if err != nil {
-		log.Fatal(err)
-	}
-	for k, branches := range all {
-		loc := perBranch[branches[0]][k]
-		data, err := loc.Blob(&k.ID)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		doc := zoekt.Document{
-			Name:              k.FullPath(),
-			Content:           data,
-			SubRepositoryPath: k.SubRepoPath,
-		}
-
-		doc.Branches = append(doc.Branches, branches...)
-		if err := builder.Add(doc); err != nil {
-			log.Printf("Add(%s): %v", doc.Name, err)
-			break
-		}
-	}
-	if err := builder.Finish(); err != nil {
-		log.Fatalf("Finish: %v", err)
-	}
-}
-
-// getManifest parses the manifest XML at the given branch/path inside a Git repository.
-func getManifest(repo *git.Repository, branch, path string) (*manifest.Manifest, error) {
-	ref, err := repo.Reference(plumbing.ReferenceName("refs/heads/"+branch), true)
-	if err != nil {
-		return nil, err
-	}
-
-	commit, err := repo.CommitObject(ref.Hash())
-	if err != nil {
-		return nil, err
-	}
-
-	tree, err := repo.TreeObject(commit.TreeHash)
-	if err != nil {
-		return nil, err
-	}
-
-	entry, err := tree.FindEntry(path)
-	if err != nil {
-		return nil, err
-	}
-
-	blob, err := repo.BlobObject(entry.Hash)
-	if err != nil {
-		return nil, err
-	}
-	r, err := blob.Reader()
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-
-	content, _ := ioutil.ReadAll(r)
-	return manifest.Parse(content)
-}
-
-// iterateManifest constructs a complete tree from the given Manifest.
-func iterateManifest(mf *manifest.Manifest,
-	baseURL url.URL, revPrefix string,
-	cache *gitindex.RepoCache) (map[fileKey]gitindex.BlobLocation, map[string]plumbing.Hash, error) {
-	allFiles := map[fileKey]gitindex.BlobLocation{}
-	allVersions := map[string]plumbing.Hash{}
-	for _, p := range mf.Project {
-		rev := mf.ProjectRevision(&p)
-
-		projURL := baseURL
-		projURL.Path = path.Join(projURL.Path, p.Name)
-
-		topRepo, err := cache.Open(&projURL)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		ref, err := topRepo.Reference(plumbing.ReferenceName(revPrefix+rev), true)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		commit, err := topRepo.CommitObject(ref.Hash())
-		if err != nil {
-			return nil, nil, err
-		}
-		if err != nil {
-			return nil, nil, err
-		}
-
-		allVersions[p.GetPath()] = commit.Hash
-
-		tree, err := commit.Tree()
-		if err != nil {
-			return nil, nil, err
-		}
-
-		files, versions, err := gitindex.TreeToFiles(topRepo, tree, projURL.String(), cache)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		for key, repo := range files {
-			allFiles[fileKey{
-				SubRepoPath: filepath.Join(p.GetPath(), key.SubRepoPath),
-				Path:        key.Path,
-				ID:          key.ID,
-			}] = repo
-		}
-
-		for path, version := range versions {
-			allVersions[filepath.Join(p.GetPath(), path)] = version
-		}
-	}
-
-	return allFiles, allVersions, nil
-}
diff --git a/cmd/zoekt-test/main.go b/cmd/zoekt-test/main.go
deleted file mode 100644
index 2d131d1..0000000
--- a/cmd/zoekt-test/main.go
+++ /dev/null
@@ -1,194 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// zoekt-test compares the search engine results with raw substring search
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"reflect"
-	"sort"
-	"strings"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/query"
-	"github.com/google/zoekt/shards"
-)
-
-func readTree(dir string) (map[string][]byte, error) {
-	var fns []string
-
-	add := func(path string, info os.FileInfo, err error) error {
-		if !info.Mode().IsRegular() {
-			return nil
-		}
-
-		fns = append(fns, path)
-		return nil
-	}
-	if err := filepath.Walk(dir, add); err != nil {
-		return nil, err
-	}
-
-	res := map[string][]byte{}
-	for _, n := range fns {
-		c, err := ioutil.ReadFile(n)
-		if err != nil {
-			return nil, err
-		}
-
-		strip := strings.TrimPrefix(n, dir+"/")
-		res[strip] = c
-	}
-	return res, nil
-}
-
-func compare(dir, patfile string, caseSensitive bool) error {
-	indexDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(indexDir)
-
-	var opts build.Options
-	opts.SetDefaults()
-	opts.IndexDir = indexDir
-
-	fileContents, err := readTree(dir)
-	if err != nil {
-		return err
-	}
-	if len(fileContents) == 0 {
-		return fmt.Errorf("no contents")
-	}
-
-	builder, err := build.NewBuilder(opts)
-	if err != nil {
-		return err
-	}
-	for k, v := range fileContents {
-		builder.AddFile(k, v)
-	}
-	if err := builder.Finish(); err != nil {
-		return err
-	}
-
-	if !caseSensitive {
-		for k, v := range fileContents {
-			fileContents[k] = toLower(v)
-		}
-	}
-
-	f, err := os.Open(patfile)
-	if err != nil {
-		return err
-	}
-	searcher, err := shards.NewDirectorySearcher(indexDir)
-	if err != nil {
-		return err
-	}
-
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		t := scanner.Text()
-		if len(t) < 3 {
-			continue
-		}
-		q := &query.Substring{
-			Pattern:       t,
-			CaseSensitive: caseSensitive,
-		}
-
-		zFiles := map[string]struct{}{}
-		rFiles := map[string]struct{}{}
-
-		// search engine results
-		var opts zoekt.SearchOptions
-		res, err := searcher.Search(context.Background(), q, &opts)
-		if err != nil {
-			return err
-		}
-
-		for _, f := range res.Files {
-			zFiles[f.FileName] = struct{}{}
-		}
-
-		// raw search
-		needle := []byte(t)
-		if !caseSensitive {
-			needle = toLower(needle)
-		}
-
-		for k, v := range fileContents {
-			if bytes.Contains(v, needle) {
-				rFiles[k] = struct{}{}
-			}
-		}
-
-		if !reflect.DeepEqual(zFiles, rFiles) {
-			var add, del []string
-			for k := range zFiles {
-				if _, ok := rFiles[k]; !ok {
-					del = append(del, k)
-				}
-			}
-			for k := range rFiles {
-				if _, ok := zFiles[k]; !ok {
-					add = append(add, k)
-				}
-			}
-			sort.Strings(add)
-			sort.Strings(del)
-			log.Printf("pattern %q, add %v, del %v", t, add, del)
-		}
-	}
-	return nil
-}
-
-func main() {
-	repo := flag.String("repo", "", "repository to search")
-	caseSensitive := flag.Bool("case", false, "case sensitive")
-	flag.Parse()
-
-	if len(flag.Args()) == 0 {
-		fmt.Fprintf(os.Stderr, "pattern file is missing.\n")
-		flag.Usage()
-		os.Exit(2)
-	}
-	input := flag.Arg(0)
-
-	if err := compare(*repo, input, *caseSensitive); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func toLower(in []byte) []byte {
-	out := make([]byte, len(in))
-	for i, c := range in {
-		if c >= 'A' && c <= 'Z' {
-			c = c - 'A' + 'a'
-		}
-		out[i] = c
-	}
-	return out
-}
diff --git a/cmd/zoekt-webserver/main.go b/cmd/zoekt-webserver/main.go
deleted file mode 100644
index 0cb4771..0000000
--- a/cmd/zoekt-webserver/main.go
+++ /dev/null
@@ -1,264 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"crypto/tls"
-	"flag"
-	"fmt"
-	"html/template"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/http/pprof"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/shards"
-	"github.com/google/zoekt/web"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"go.uber.org/automaxprocs/maxprocs"
-	"golang.org/x/net/trace"
-)
-
-const logFormat = "2006-01-02T15-04-05.999999999Z07"
-
-func divertLogs(dir string, interval time.Duration) {
-	t := time.NewTicker(interval)
-	var last *os.File
-	for {
-		nm := filepath.Join(dir, fmt.Sprintf("zoekt-webserver.%s.%d.log", time.Now().Format(logFormat), os.Getpid()))
-		fmt.Fprintf(os.Stderr, "writing logs to %s\n", nm)
-
-		f, err := os.Create(nm)
-		if err != nil {
-			// There is not much we can do now.
-			fmt.Fprintf(os.Stderr, "can't create output file %s: %v\n", nm, err)
-			os.Exit(2)
-		}
-
-		log.SetOutput(f)
-		last.Close()
-
-		last = f
-
-		<-t.C
-	}
-}
-
-const templateExtension = ".html.tpl"
-
-func loadTemplates(tpl *template.Template, dir string) error {
-	fs, err := filepath.Glob(dir + "/*" + templateExtension)
-	if err != nil {
-		log.Fatalf("Glob: %v", err)
-	}
-
-	log.Printf("loading templates: %v", fs)
-	for _, fn := range fs {
-		content, err := ioutil.ReadFile(fn)
-		if err != nil {
-			return err
-		}
-
-		base := filepath.Base(fn)
-		base = strings.TrimSuffix(base, templateExtension)
-		if _, err := tpl.New(base).Parse(string(content)); err != nil {
-			return fmt.Errorf("template.Parse(%s): %v", fn, err)
-		}
-	}
-	return nil
-}
-
-func writeTemplates(dir string) error {
-	if dir == "" {
-		return fmt.Errorf("must set --template_dir")
-	}
-
-	for k, v := range web.TemplateText {
-		nm := filepath.Join(dir, k+templateExtension)
-		if err := ioutil.WriteFile(nm, []byte(v), 0o644); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func main() {
-	logDir := flag.String("log_dir", "", "log to this directory rather than stderr.")
-	logRefresh := flag.Duration("log_refresh", 24*time.Hour, "if using --log_dir, start writing a new file this often.")
-
-	listen := flag.String("listen", ":6070", "listen on this address.")
-	index := flag.String("index", build.DefaultDir, "set index directory to use")
-	html := flag.Bool("html", true, "enable HTML interface")
-	print := flag.Bool("print", false, "enable local result URLs")
-	enablePprof := flag.Bool("pprof", false, "set to enable remote profiling.")
-	sslCert := flag.String("ssl_cert", "", "set path to SSL .pem holding certificate.")
-	sslKey := flag.String("ssl_key", "", "set path to SSL .pem holding key.")
-	hostCustomization := flag.String(
-		"host_customization", "",
-		"specify host customization, as HOST1=QUERY,HOST2=QUERY")
-
-	templateDir := flag.String("template_dir", "", "set directory from which to load custom .html.tpl template files")
-	dumpTemplates := flag.Bool("dump_templates", false, "dump templates into --template_dir and exit.")
-	version := flag.Bool("version", false, "Print version number")
-	flag.Parse()
-
-	if *version {
-		fmt.Printf("zoekt-webserver version %q\n", zoekt.Version)
-		os.Exit(0)
-	}
-
-	if *dumpTemplates {
-		if err := writeTemplates(*templateDir); err != nil {
-			log.Fatal(err)
-		}
-		os.Exit(0)
-	}
-
-	if *logDir != "" {
-		if fi, err := os.Lstat(*logDir); err != nil || !fi.IsDir() {
-			log.Fatalf("%s is not a directory", *logDir)
-		}
-		// We could do fdup acrobatics to also redirect
-		// stderr, but it is simpler and more portable for the
-		// caller to divert stderr output if necessary.
-		go divertLogs(*logDir, *logRefresh)
-	}
-
-	// Tune GOMAXPROCS to match Linux container CPU quota.
-	maxprocs.Set()
-
-	if err := os.MkdirAll(*index, 0o755); err != nil {
-		log.Fatal(err)
-	}
-
-	searcher, err := shards.NewDirectorySearcher(*index)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	s := &web.Server{
-		Searcher: searcher,
-		Top:      web.Top,
-		Version:  zoekt.Version,
-	}
-
-	if *templateDir != "" {
-		if err := loadTemplates(s.Top, *templateDir); err != nil {
-			log.Fatalf("loadTemplates: %v", err)
-		}
-	}
-
-	s.Print = *print
-	s.HTML = *html
-
-	if *hostCustomization != "" {
-		s.HostCustomQueries = map[string]string{}
-		for _, h := range strings.SplitN(*hostCustomization, ",", -1) {
-			if len(h) == 0 {
-				continue
-			}
-			fields := strings.SplitN(h, "=", 2)
-			if len(fields) < 2 {
-				log.Fatalf("invalid host_customization %q", h)
-			}
-
-			s.HostCustomQueries[fields[0]] = fields[1]
-		}
-	}
-
-	handler, err := web.NewMux(s)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	handler.Handle("/metrics", promhttp.Handler())
-
-	if *enablePprof {
-		handler.HandleFunc("/debug/pprof/", pprof.Index)
-		handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-		handler.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		handler.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		handler.HandleFunc("/debug/requests/", trace.Traces)
-		handler.HandleFunc("/debug/events/", trace.Events)
-	}
-
-	watchdogAddr := "http://" + *listen
-	if *sslCert != "" || *sslKey != "" {
-		watchdogAddr = "https://" + *listen
-	}
-	go watchdog(30*time.Second, watchdogAddr)
-
-	if *sslCert != "" || *sslKey != "" {
-		log.Printf("serving HTTPS on %s", *listen)
-		err = http.ListenAndServeTLS(*listen, *sslCert, *sslKey, handler)
-	} else {
-		log.Printf("serving HTTP on %s", *listen)
-		err = http.ListenAndServe(*listen, handler)
-	}
-	log.Printf("ListenAndServe: %v", err)
-}
-
-func watchdogOnce(ctx context.Context, client *http.Client, addr string) error {
-	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(5*time.Second))
-	defer cancel()
-
-	req, err := http.NewRequest("GET", addr, nil)
-	if err != nil {
-		return err
-	}
-
-	req = req.WithContext(ctx)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("watchdog: status %v", resp.StatusCode)
-	}
-	return nil
-}
-
-func watchdog(dt time.Duration, addr string) {
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{
-		Transport: tr,
-	}
-	tick := time.NewTicker(dt)
-
-	errCount := 0
-	for range tick.C {
-		err := watchdogOnce(context.Background(), client, addr)
-		if err != nil {
-			errCount++
-		} else {
-			errCount = 0
-		}
-		if errCount == 3 {
-			log.Panicf("watchdog: %v", err)
-		}
-	}
-}
diff --git a/cmd/zoekt/main.go b/cmd/zoekt/main.go
deleted file mode 100644
index 035e8fa..0000000
--- a/cmd/zoekt/main.go
+++ /dev/null
@@ -1,158 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"runtime/pprof"
-	"time"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/query"
-	"github.com/google/zoekt/shards"
-)
-
-func displayMatches(files []zoekt.FileMatch, pat string, withRepo bool, list bool) {
-	for _, f := range files {
-		r := ""
-		if withRepo {
-			r = f.Repository + "/"
-		}
-		if list {
-			fmt.Printf("%s%s\n", r, f.FileName)
-			continue
-		}
-
-		for _, m := range f.LineMatches {
-			fmt.Printf("%s%s:%d:%s\n", r, f.FileName, m.LineNumber, m.Line)
-		}
-	}
-}
-
-func loadShard(fn string, verbose bool) (zoekt.Searcher, error) {
-	f, err := os.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-
-	iFile, err := zoekt.NewIndexFile(f)
-	if err != nil {
-		return nil, err
-	}
-
-	s, err := zoekt.NewSearcher(iFile)
-	if err != nil {
-		iFile.Close()
-		return nil, fmt.Errorf("NewSearcher(%s): %v", fn, err)
-	}
-
-	if verbose {
-		repo, index, err := zoekt.ReadMetadata(iFile)
-		if err != nil {
-			iFile.Close()
-			return nil, fmt.Errorf("ReadMetadata(%s): %v", fn, err)
-		}
-		log.Printf("repo metadata: %#v", repo)
-		log.Printf("index metadata: %#v", index)
-	}
-
-	return s, nil
-}
-
-func main() {
-	shard := flag.String("shard", "", "search in a specific shard")
-	index := flag.String("index_dir",
-		filepath.Join(os.Getenv("HOME"), ".zoekt"), "search for index files in `directory`")
-	cpuProfile := flag.String("cpu_profile", "", "write cpu profile to `file`")
-	profileTime := flag.Duration("profile_time", time.Second, "run this long to gather stats.")
-	verbose := flag.Bool("v", false, "print some background data")
-	withRepo := flag.Bool("r", false, "print the repo before the file name")
-	list := flag.Bool("l", false, "print matching filenames only")
-
-	flag.Usage = func() {
-		name := os.Args[0]
-		fmt.Fprintf(os.Stderr, "Usage:\n\n  %s [option] QUERY\n"+
-			"for example\n\n  %s 'byte file:java -file:test'\n\n", name, name)
-		flag.PrintDefaults()
-		fmt.Fprintf(os.Stderr, "\n")
-	}
-	flag.Parse()
-
-	if len(flag.Args()) == 0 {
-		fmt.Fprintf(os.Stderr, "Pattern is missing.\n")
-		flag.Usage()
-		os.Exit(2)
-	}
-	pat := flag.Arg(0)
-
-	var searcher zoekt.Searcher
-	var err error
-	if *shard != "" {
-		searcher, err = loadShard(*shard, *verbose)
-	} else {
-		searcher, err = shards.NewDirectorySearcher(*index)
-	}
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	query, err := query.Parse(pat)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if *verbose {
-		log.Println("query:", query)
-	}
-
-	var sOpts zoekt.SearchOptions
-	sres, err := searcher.Search(context.Background(), query, &sOpts)
-	if *cpuProfile != "" {
-		// If profiling, do it another time so we measure with
-		// warm caches.
-		f, err := os.Create(*cpuProfile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		defer f.Close()
-		if *verbose {
-			log.Println("Displaying matches...")
-		}
-
-		t := time.Now()
-		pprof.StartCPUProfile(f)
-		for {
-			sres, _ = searcher.Search(context.Background(), query, &sOpts)
-			if time.Since(t) > *profileTime {
-				break
-			}
-		}
-		pprof.StopCPUProfile()
-	}
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	displayMatches(sres.Files, pat, *withRepo, *list)
-	if *verbose {
-		log.Printf("stats: %#v", sres.Stats)
-	}
-}
diff --git a/contentprovider.go b/contentprovider.go
deleted file mode 100644
index 62fd149..0000000
--- a/contentprovider.go
+++ /dev/null
@@ -1,310 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"bytes"
-	"log"
-	"sort"
-	"unicode/utf8"
-)
-
-var _ = log.Println
-
-// contentProvider is an abstraction to treat matches for names and
-// content with the same code.
-type contentProvider struct {
-	id    *indexData
-	stats *Stats
-
-	// mutable
-	err      error
-	idx      uint32
-	_data    []byte
-	_nl      []uint32
-	_nlBuf   []uint32
-	_sects   []DocumentSection
-	_sectBuf []DocumentSection
-	fileSize uint32
-}
-
-// setDocument skips to the given document.
-func (p *contentProvider) setDocument(docID uint32) {
-	fileStart := p.id.boundaries[docID]
-
-	p.idx = docID
-	p.fileSize = p.id.boundaries[docID+1] - fileStart
-
-	p._nl = nil
-	p._sects = nil
-	p._data = nil
-}
-
-func (p *contentProvider) docSections() []DocumentSection {
-	if p._sects == nil {
-		var sz uint32
-		p._sects, sz, p.err = p.id.readDocSections(p.idx, p._sectBuf)
-		p.stats.ContentBytesLoaded += int64(sz)
-		p._sectBuf = p._sects
-	}
-	return p._sects
-}
-
-func (p *contentProvider) newlines() []uint32 {
-	if p._nl == nil {
-		var sz uint32
-		p._nl, sz, p.err = p.id.readNewlines(p.idx, p._nlBuf)
-		p._nlBuf = p._nl
-		p.stats.ContentBytesLoaded += int64(sz)
-	}
-	return p._nl
-}
-
-func (p *contentProvider) data(fileName bool) []byte {
-	if fileName {
-		return p.id.fileNameContent[p.id.fileNameIndex[p.idx]:p.id.fileNameIndex[p.idx+1]]
-	}
-
-	if p._data == nil {
-		p._data, p.err = p.id.readContents(p.idx)
-		p.stats.FilesLoaded++
-		p.stats.ContentBytesLoaded += int64(len(p._data))
-	}
-	return p._data
-}
-
-// Find offset in bytes (relative to corpus start) for an offset in
-// runes (relative to document start). If filename is set, the corpus
-// is the set of filenames, with the document being the name itself.
-func (p *contentProvider) findOffset(filename bool, r uint32) uint32 {
-	if p.id.metaData.PlainASCII {
-		return r
-	}
-
-	sample := p.id.runeOffsets
-	runeEnds := p.id.fileEndRunes
-	fileStartByte := p.id.boundaries[p.idx]
-	if filename {
-		sample = p.id.fileNameRuneOffsets
-		runeEnds = p.id.fileNameEndRunes
-		fileStartByte = p.id.fileNameIndex[p.idx]
-	}
-
-	absR := r
-	if p.idx > 0 {
-		absR += runeEnds[p.idx-1]
-	}
-
-	byteOff := sample[absR/runeOffsetFrequency]
-	left := absR % runeOffsetFrequency
-
-	var data []byte
-
-	if filename {
-		data = p.id.fileNameContent[byteOff:]
-	} else {
-		data, p.err = p.id.readContentSlice(byteOff, 3*runeOffsetFrequency)
-		if p.err != nil {
-			return 0
-		}
-	}
-	for left > 0 {
-		_, sz := utf8.DecodeRune(data)
-		byteOff += uint32(sz)
-		data = data[sz:]
-		left--
-	}
-
-	byteOff -= fileStartByte
-	return byteOff
-}
-
-func (p *contentProvider) fillMatches(ms []*candidateMatch) []LineMatch {
-	var result []LineMatch
-	if ms[0].fileName {
-		// There is only "line" in a filename.
-		res := LineMatch{
-			Line:     p.id.fileName(p.idx),
-			FileName: true,
-		}
-
-		for _, m := range ms {
-			res.LineFragments = append(res.LineFragments, LineFragmentMatch{
-				LineOffset:  int(m.byteOffset),
-				MatchLength: int(m.byteMatchSz),
-				Offset:      m.byteOffset,
-			})
-
-			result = []LineMatch{res}
-		}
-	} else {
-		ms = breakMatchesOnNewlines(ms, p.data(false))
-		result = p.fillContentMatches(ms)
-	}
-
-	sects := p.docSections()
-	for i, m := range result {
-		result[i].Score = matchScore(sects, &m)
-	}
-
-	return result
-}
-
-func (p *contentProvider) fillContentMatches(ms []*candidateMatch) []LineMatch {
-	var result []LineMatch
-	for len(ms) > 0 {
-		m := ms[0]
-		num, lineStart, lineEnd := m.line(p.newlines(), p.fileSize)
-
-		var lineCands []*candidateMatch
-
-		endMatch := m.byteOffset + m.byteMatchSz
-
-		for len(ms) > 0 {
-			m := ms[0]
-			if int(m.byteOffset) <= lineEnd {
-				endMatch = m.byteOffset + m.byteMatchSz
-				lineCands = append(lineCands, m)
-				ms = ms[1:]
-			} else {
-				break
-			}
-		}
-
-		if len(lineCands) == 0 {
-			log.Panicf(
-				"%s %v infinite loop: num %d start,end %d,%d, offset %d",
-				p.id.fileName(p.idx), p.id.metaData,
-				num, lineStart, lineEnd,
-				m.byteOffset)
-		}
-
-		data := p.data(false)
-
-		// Due to merging matches, we may have a match that
-		// crosses a line boundary. Prevent confusion by
-		// taking lines until we pass the last match
-		for lineEnd < len(data) && endMatch > uint32(lineEnd) {
-			next := bytes.IndexByte(data[lineEnd+1:], '\n')
-			if next == -1 {
-				lineEnd = len(data)
-			} else {
-				// TODO(hanwen): test that checks "+1" part here.
-				lineEnd += next + 1
-			}
-		}
-
-		finalMatch := LineMatch{
-			LineStart:  lineStart,
-			LineEnd:    lineEnd,
-			LineNumber: num,
-		}
-		finalMatch.Line = data[lineStart:lineEnd]
-
-		for _, m := range lineCands {
-			fragment := LineFragmentMatch{
-				Offset:      m.byteOffset,
-				LineOffset:  int(m.byteOffset) - lineStart,
-				MatchLength: int(m.byteMatchSz),
-			}
-			finalMatch.LineFragments = append(finalMatch.LineFragments, fragment)
-		}
-		result = append(result, finalMatch)
-	}
-	return result
-}
-
-const (
-	// TODO - how to scale this relative to rank?
-	scorePartialWordMatch   = 50.0
-	scoreWordMatch          = 500.0
-	scoreImportantThreshold = 2000.0
-	scorePartialSymbol      = 4000.0
-	scoreSymbol             = 7000.0
-	scoreFactorAtomMatch    = 400.0
-	scoreShardRankFactor    = 20.0
-	scoreFileOrderFactor    = 10.0
-	scoreLineOrderFactor    = 1.0
-)
-
-func findSection(secs []DocumentSection, off, sz uint32) *DocumentSection {
-	j := sort.Search(len(secs), func(i int) bool {
-		return secs[i].End >= off+sz
-	})
-
-	if j == len(secs) {
-		return nil
-	}
-
-	if secs[j].Start <= off && off+sz <= secs[j].End {
-		return &secs[j]
-	}
-	return nil
-}
-
-func matchScore(secs []DocumentSection, m *LineMatch) float64 {
-	var maxScore float64
-	for _, f := range m.LineFragments {
-		startBoundary := f.LineOffset < len(m.Line) && (f.LineOffset == 0 || byteClass(m.Line[f.LineOffset-1]) != byteClass(m.Line[f.LineOffset]))
-
-		end := int(f.LineOffset) + f.MatchLength
-		endBoundary := end > 0 && (end == len(m.Line) || byteClass(m.Line[end-1]) != byteClass(m.Line[end]))
-
-		score := 0.0
-		if startBoundary && endBoundary {
-			score = scoreWordMatch
-		} else if startBoundary || endBoundary {
-			score = scorePartialWordMatch
-		}
-
-		sec := findSection(secs, f.Offset, uint32(f.MatchLength))
-		if sec != nil {
-			startMatch := sec.Start == f.Offset
-			endMatch := sec.End == f.Offset+uint32(f.MatchLength)
-			if startMatch && endMatch {
-				score += scoreSymbol
-			} else if startMatch || endMatch {
-				score += (scoreSymbol + scorePartialSymbol) / 2
-			} else {
-				score += scorePartialSymbol
-			}
-		}
-		if score > maxScore {
-			maxScore = score
-		}
-	}
-	return maxScore
-}
-
-type matchScoreSlice []LineMatch
-
-func (m matchScoreSlice) Len() int           { return len(m) }
-func (m matchScoreSlice) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
-func (m matchScoreSlice) Less(i, j int) bool { return m[i].Score > m[j].Score }
-
-type fileMatchSlice []FileMatch
-
-func (m fileMatchSlice) Len() int           { return len(m) }
-func (m fileMatchSlice) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
-func (m fileMatchSlice) Less(i, j int) bool { return m[i].Score > m[j].Score }
-
-func sortMatchesByScore(ms []LineMatch) {
-	sort.Sort(matchScoreSlice(ms))
-}
-
-// Sort a slice of results.
-func SortFilesByScore(ms []FileMatch) {
-	sort.Sort(fileMatchSlice(ms))
-}
diff --git a/ctags/json.go b/ctags/json.go
deleted file mode 100644
index 980b54b..0000000
--- a/ctags/json.go
+++ /dev/null
@@ -1,265 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ctags
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"os/exec"
-	"runtime"
-	"strings"
-	"sync"
-)
-
-const debug = false
-
-type ctagsProcess struct {
-	cmd     *exec.Cmd
-	in      io.WriteCloser
-	out     *scanner
-	outPipe io.ReadCloser
-}
-
-func newProcess(bin string) (*ctagsProcess, error) {
-	opt := "default"
-	if runtime.GOOS == "linux" {
-		opt = "sandbox"
-	}
-
-	cmd := exec.Command(bin, "--_interactive="+opt, "--fields=*")
-	in, err := cmd.StdinPipe()
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := cmd.StdoutPipe()
-	if err != nil {
-		in.Close()
-		return nil, err
-	}
-	cmd.Stderr = os.Stderr
-	proc := ctagsProcess{
-		cmd:     cmd,
-		in:      in,
-		out:     &scanner{r: bufio.NewReaderSize(out, 4096)},
-		outPipe: out,
-	}
-
-	if err := cmd.Start(); err != nil {
-		return nil, err
-	}
-
-	var init reply
-	if err := proc.read(&init); err != nil {
-		return nil, err
-	}
-
-	return &proc, nil
-}
-
-func (p *ctagsProcess) Close() {
-	p.cmd.Process.Kill()
-	p.outPipe.Close()
-	p.in.Close()
-}
-
-func (p *ctagsProcess) read(rep *reply) error {
-	if !p.out.Scan() {
-		// Some errors do not kill the parser. We would deadlock if we waited
-		// for the process to exit.
-		err := p.out.Err()
-		p.Close()
-		return err
-	}
-	if debug {
-		log.Printf("read %q", p.out.Bytes())
-	}
-
-	// See https://github.com/universal-ctags/ctags/issues/1493
-	if bytes.Equal([]byte("(null)"), p.out.Bytes()) {
-		return nil
-	}
-
-	err := json.Unmarshal(p.out.Bytes(), rep)
-	if err != nil {
-		return fmt.Errorf("unmarshal(%q): %v", p.out.Bytes(), err)
-	}
-	return nil
-}
-
-func (p *ctagsProcess) post(req *request, content []byte) error {
-	body, err := json.Marshal(req)
-	if err != nil {
-		return err
-	}
-	body = append(body, '\n')
-	if debug {
-		log.Printf("post %q", body)
-	}
-
-	if _, err = p.in.Write(body); err != nil {
-		return err
-	}
-	_, err = p.in.Write(content)
-	if debug {
-		log.Println(string(content))
-	}
-	return err
-}
-
-type request struct {
-	Command  string `json:"command"`
-	Filename string `json:"filename"`
-	Size     int    `json:"size"`
-}
-
-type reply struct {
-	// Init
-	Typ     string `json:"_type"`
-	Name    string `json:"name"`
-	Version string `json:"version"`
-
-	// completed
-	Command string `json:"command"`
-
-	// Ignore pattern: we don't use it and universal-ctags
-	// sometimes generates 'false' as value.
-	Path      string `json:"path"`
-	Language  string `json:"language"`
-	Line      int    `json:"line"`
-	Kind      string `json:"kind"`
-	End       int    `json:"end"`
-	Scope     string `json:"scope"`
-	ScopeKind string `json:"scopeKind"`
-	Access    string `json:"access"`
-	Signature string `json:"signature"`
-}
-
-func (p *ctagsProcess) Parse(name string, content []byte) ([]*Entry, error) {
-	req := request{
-		Command:  "generate-tags",
-		Size:     len(content),
-		Filename: name,
-	}
-
-	if err := p.post(&req, content); err != nil {
-		return nil, err
-	}
-
-	var es []*Entry
-	for {
-		var rep reply
-		if err := p.read(&rep); err != nil {
-			return nil, err
-		}
-		if rep.Typ == "completed" {
-			break
-		}
-
-		e := Entry{
-			Sym:      rep.Name,
-			Path:     rep.Path,
-			Line:     rep.Line,
-			Kind:     rep.Kind,
-			Language: rep.Language,
-		}
-
-		es = append(es, &e)
-	}
-
-	return es, nil
-}
-
-// scanner is like bufio.Scanner but skips long lines instead of returning
-// bufio.ErrTooLong.
-//
-// Additionally it will skip empty lines.
-type scanner struct {
-	r    *bufio.Reader
-	line []byte
-	err  error
-}
-
-func (s *scanner) Scan() bool {
-	if s.err != nil {
-		return false
-	}
-
-	var (
-		err  error
-		line []byte
-	)
-
-	for err == nil && len(line) == 0 {
-		line, err = s.r.ReadSlice('\n')
-		for err == bufio.ErrBufferFull {
-			// make line empty so we ignore it
-			line = nil
-			_, err = s.r.ReadSlice('\n')
-		}
-		line = bytes.TrimSuffix(line, []byte{'\n'})
-		line = bytes.TrimSuffix(line, []byte{'\r'})
-	}
-
-	s.line, s.err = line, err
-	return len(line) > 0
-}
-
-func (s *scanner) Bytes() []byte {
-	return s.line
-}
-
-func (s *scanner) Err() error {
-	if s.err == io.EOF {
-		return nil
-	}
-	return s.err
-}
-
-type Parser interface {
-	Parse(name string, content []byte) ([]*Entry, error)
-}
-
-type lockedParser struct {
-	p Parser
-	l sync.Mutex
-}
-
-func (lp *lockedParser) Parse(name string, content []byte) ([]*Entry, error) {
-	lp.l.Lock()
-	defer lp.l.Unlock()
-	return lp.p.Parse(name, content)
-}
-
-// NewParser creates a parser that is implemented by the given
-// universal-ctags binary. The parser is safe for concurrent use.
-func NewParser(bin string) (Parser, error) {
-	if strings.Contains(bin, "universal-ctags") {
-		// todo: restart, parallelization.
-		proc, err := newProcess(bin)
-		if err != nil {
-			return nil, err
-		}
-		return &lockedParser{p: proc}, nil
-	}
-
-	log.Fatal("not implemented")
-	return nil, nil
-}
diff --git a/ctags/json_test.go b/ctags/json_test.go
deleted file mode 100644
index c6f1d3c..0000000
--- a/ctags/json_test.go
+++ /dev/null
@@ -1,143 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ctags
-
-import (
-	"bufio"
-	"os/exec"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-)
-
-func TestJSON(t *testing.T) {
-	if _, err := exec.LookPath("universal-ctags"); err != nil {
-		t.Skip(err)
-	}
-
-	p, err := newProcess("universal-ctags")
-	if err != nil {
-		t.Fatal("newProcess", err)
-	}
-
-	defer p.Close()
-
-	java := `
-package io.zoekt;
-import java.util.concurrent.Future;
-class Back implements Future extends Frob {
-  public static int BLA = 1;
-  public int member;
-  public Back() {
-    member = 2;
-  }
-  public int method() {
-    member++;
-  }
-}
-`
-	name := "io/zoekt/Back.java"
-	got, err := p.Parse(name, []byte(java))
-	if err != nil {
-		t.Errorf("Process: %v", err)
-	}
-
-	want := []*Entry{
-		{
-			Sym:      "io.zoekt",
-			Kind:     "package",
-			Language: "Java",
-			Path:     "io/zoekt/Back.java",
-			Line:     2,
-		},
-		{
-			Sym:      "Back",
-			Path:     "io/zoekt/Back.java",
-			Line:     4,
-			Language: "Java",
-			Kind:     "class",
-		},
-
-		{
-			Sym:      "BLA",
-			Path:     "io/zoekt/Back.java",
-			Line:     5,
-			Kind:     "field",
-			Language: "Java",
-		},
-		{
-			Sym:      "member",
-			Path:     "io/zoekt/Back.java",
-			Line:     6,
-			Language: "Java",
-			Kind:     "field",
-		},
-		{
-			Sym:      "Back",
-			Path:     "io/zoekt/Back.java",
-			Language: "Java",
-			Line:     7,
-			Kind:     "method",
-		},
-		{
-			Sym:      "method",
-			Language: "Java",
-			Path:     "io/zoekt/Back.java",
-			Line:     10,
-			Kind:     "method",
-		},
-	}
-
-	for i := range want {
-		if !reflect.DeepEqual(got[i], want[i]) {
-			t.Fatalf("got %#v, want %#v", got[i], want[i])
-		}
-	}
-}
-
-func TestScanner(t *testing.T) {
-	size := 20
-
-	input := strings.Join([]string{
-		"aaaaaaaaa",
-		strings.Repeat("B", 3*size+3),
-		strings.Repeat("C", size) + strings.Repeat("D", size+1),
-		"",
-		strings.Repeat("e", size-1),
-		"f\r",
-		"gg",
-	}, "\n")
-	want := []string{
-		"aaaaaaaaa",
-		strings.Repeat("e", size-1),
-		"f",
-		"gg",
-	}
-
-	var got []string
-	r := &scanner{r: bufio.NewReaderSize(strings.NewReader(input), size)}
-	for r.Scan() {
-		got = append(got, string(r.Bytes()))
-	}
-	if err := r.Err(); err != nil {
-		t.Fatal(err)
-	}
-
-	if !cmp.Equal(got, want) {
-		t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(want, got))
-	}
-}
diff --git a/ctags/parse.go b/ctags/parse.go
deleted file mode 100644
index d199428..0000000
--- a/ctags/parse.go
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ctags
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-)
-
-type Entry struct {
-	Sym        string
-	Path       string
-	Line       int
-	Kind       string
-	Language   string
-	Parent     string
-	ParentType string
-
-	FileLimited bool
-}
-
-// Parse parses a single line of exuberant "ctags -n" output.
-func Parse(in string) (*Entry, error) {
-	fields := strings.Split(in, "\t")
-	e := Entry{}
-
-	if len(fields) < 3 {
-		return nil, fmt.Errorf("too few fields: %q", in)
-	}
-
-	e.Sym = fields[0]
-	e.Path = fields[1]
-
-	lstr := fields[2]
-	if len(lstr) < 2 {
-		return nil, fmt.Errorf("got %q for linenum field", lstr)
-	}
-
-	l, err := strconv.ParseInt(lstr[:len(lstr)-2], 10, 64)
-	if err != nil {
-		return nil, err
-	}
-	e.Line = int(l)
-	e.Kind = fields[3]
-
-field:
-	for _, f := range fields[3:] {
-		if string(f) == "file:" {
-			e.FileLimited = true
-		}
-		for _, p := range []string{"class", "enum"} {
-			if strings.HasPrefix(f, p+":") {
-				e.Parent = strings.TrimPrefix(f, p+":")
-				e.ParentType = p
-				continue field
-			}
-		}
-	}
-	return &e, nil
-}
diff --git a/ctags/parse_test.go b/ctags/parse_test.go
deleted file mode 100644
index 30f41a1..0000000
--- a/ctags/parse_test.go
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ctags
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestParse(t *testing.T) {
-	type testcase struct {
-		in  string
-		out *Entry
-	}
-	cases := []testcase{
-		{
-			`ABBREV_SHA	./gitiles-servlet/src/main/java/com/google/gitiles/CommitData.java	59;"	e	enum:CommitData.Field	file:`,
-			&Entry{
-				Sym:         "ABBREV_SHA",
-				Path:        "./gitiles-servlet/src/main/java/com/google/gitiles/CommitData.java",
-				Line:        59,
-				Kind:        "e",
-				Parent:      "CommitData.Field",
-				ParentType:  "enum",
-				FileLimited: true,
-			},
-		},
-		{
-			`ACCESS_ATTRIBUTE	./gitiles-servlet/src/main/java/com/google/gitiles/CommitData.java	55;"	f	class:BaseServlet	file:`,
-			&Entry{
-				Sym:         "ACCESS_ATTRIBUTE",
-				Path:        "./gitiles-servlet/src/main/java/com/google/gitiles/CommitData.java",
-				Line:        55,
-				Kind:        "f",
-				Parent:      "BaseServlet",
-				ParentType:  "class",
-				FileLimited: true,
-			},
-		},
-	}
-	for _, c := range cases {
-		e, err := Parse(c.in)
-		if err != nil && c.out != nil {
-			t.Errorf("Parse(%s): %v", c.in, err)
-		} else if !reflect.DeepEqual(c.out, e) {
-			t.Errorf("Parse(%s): got %#v, want %#v", c.in, e, c.out)
-		}
-	}
-}
diff --git a/doc/ctags.md b/doc/ctags.md
deleted file mode 100644
index 36b4014..0000000
--- a/doc/ctags.md
+++ /dev/null
@@ -1,41 +0,0 @@
-
-CTAGS
-=====
-
-Ctags generates indices of symbol definitions in source files. It
-started its life as part of the BSD Unix, but there are several more
-modern flavors. Zoekt supports both [exuberant
-ctags](http://ctags.sourceforge.net/) and
-[universal-ctags](https://github.com/universal-ctags).
-
-It is strongly recommended to use Universal Ctags, [version
-`db3d9a6`](https://github.com/universal-ctags/ctags/commit/4ff09da9b0a36a9e75c92f4be05d476b35b672cd)
-or newer, running on the Linux platform.
-
-From this version on, universal ctags will be called using seccomp,
-which guarantees that security problems in ctags cannot escalate to
-access to the indexing machine.
-
-Ubuntu, Debian and Arch provide universal ctags with seccomp support
-compiled in. Zoekt expects the `universal-ctags` binary to be on
-`$PATH`. Note: only Ubuntu names the binary `universal-ctags`, while
-most distributions name it `ctags`.
-
-Use the following invocation to compile and install universal-ctags:
-
-```
-sudo apt-get install
-  pkg-config autoconf \
-  libseccomp-dev libseccomp \
-  libjansson-dev libjansson 
-
-./autogen.sh
-LDFLAGS=-static ./configure --enable-json --enable-seccomp
-make -j4
-
-# create tarball
-NAME=ctags-$(date --iso-8601=minutes | tr -d ':' | sed 's|\+.*$||')-$(git show --pretty=format:%h -q)
-mkdir ${NAME}
-cp ctags ${NAME}/universal-ctags
-tar zcf ${NAME}.tar.gz ${NAME}/
-```
diff --git a/doc/design.md b/doc/design.md
deleted file mode 100644
index df3bfea..0000000
--- a/doc/design.md
+++ /dev/null
@@ -1,328 +0,0 @@
-
-
-OBJECTIVE
-=========
-
-Provide full text code search for git based corpuses.
-
-Goals:
-
-* sub-50ms results on large codebases, such as Android (~2G text) or
-  Chrome
-
-* works well on a single standard Linux machine, with stable storage on SSD
-
-* search multiple repositories and multiple branches.
-
-* provide rich query language, with boolean operators
-
-* integrate with Gerrit/Gitiles code review/browsing system
-
-
-SEARCHING AND INDEXING
-======================
-
-
-Positional trigrams
--------------------
-
-We build an index of ngrams (n=3), where we store the offset of each
-ngram's occurrence within a file.  For example, if the corpus is "banana"
-then we generate the index
-
-    "ban": 0
-    "ana": 1,3
-    "nan": 2
-
-If we are searching for a string (eg. "The quick brown fox"), then we
-look for two trigrams (eg. "The" and "fox"), and check that they are
-found at the right distance apart.
-
-Regular expressions are handled by extracting normal strings from the regular
-expressions. For example, to search for
-
-    (Path|PathFragment).*=.*/usr/local
-
-we look for
-
-    (AND (OR substr:"Path" substr:"PathFragment") substr:"/usr/local")
-
-and any documents thus found would be searched for the regular
-expression.
-
-Compared to indexing 3-grams on a per-file basis, as described
-[here](https://swtch.com/~rsc/regexp/regexp4.html), there are some advantages:
-
-* for each substring, we only have to intersect just a couple of posting-lists:
-  one for the beginning, and one for the end.
-
-* Since we touch few posting lists per query, they can be stored on
-  slower media, such as SSD.
-
-* we can select any pair of trigrams from the pattern for which the
-  number of matches is minimal. For example, we could search for "qui"
-  rather than "the".
-
-There are some downsides compared to trigrams:
-
-* The index is large. Empirically, it is about 3x the corpus size, composed of
-  2x (offsets), and 1x (original content). However, since we have to look at
-  just a limited number of ngrams, we don't have to keep the index in memory.
-
-Compared to [suffix
-arrays](https://blog.nelhage.com/2015/02/regular-expression-search-with-suffix-arrays/),
-there are the following advantages:
-
-* The index construction is straightforward, and can easily be made
-  incremental.
-
-* Since the posting lists for a trigram can be stored on SSD,
-  searching with positional trigrams only requires 1.2x corpus size of
-  RAM.
-
-* All the matches are returned in document order. This makes it
-  straightforward to process compound boolean queries with AND and OR.
-
-Downsides compared to suffix array:
-
-* there is no way to transform regular expressions into index ranges into
-  the suffix array.
-
-
-Case sensitivity
-----------------
-
-Code usually is searched without regard for case. In this case, when
-we are looking for "abc", we look for occurrences of all the different
-case variants, ie. {"abc", "Abc", "aBc", "ABc", ... }, and then
-compare the candidate matches without regard for case.
-
-
-UTF-8
------
-
-UTF-8 is the defacto encoding for unicode. Zoekt assumes that files
-are UTF-8 encoded.  Characters have differing widths in UTF-8, so we
-use rune offsets in the trigram index, and convert those back to bytes
-with a lookup table: every 100 runes, we store the rune-index to
-byte-index mapping. For corpuses that are completely ASCII (fairly
-normal for source code), we short-circuit this lookup.
-
-
-Branches
---------
-
-Each file blob in the index has a bitmask, representing the branches
-in which the content is found, eg:
-
-    branches: [master=1, staging=2, stable=4]
-    file "x.java", branch mask=3
-    file "x.java", branch mask=4
-
-in this case, the index holds two versions of "x.java", the one
-present in "master" and "staging", and the one in the "stable" branch.
-
-With this technique, we can index many similar branches of a
-repository with little space overhead.
-
-
-Index format
-------------
-
-The index is organized in shards, where each shard is a file, laid out
-such that it can be mmap'd efficiently.
-
-Each shard contains data for one code repository. The basic data in an
-index shard are the following
-
-   * file contents
-   * filenames
-   * the content posting lists (varint encoded)
-   * the filename posting lists (varint encoded)
-   * branch masks
-   * metadata (repository name, index format version, etc.)
-
-In practice, the shard size is about 3x the corpus (size).
-
-The format uses uint32 for all offsets, so the total size of a shard
-should be below 4G. Given the size of the posting data, this caps
-content size per shard at 1G.
-
-Currently, within a shard, a single goroutine searches all documents,
-so the shard size determines the amount of parallelism, and large
-repositories should be split across multiple shards to achieve good
-performance.
-
-The metadata section contains a version number (which by convention is
-also part of the file name of the shard). This provides a smooth
-upgrade path across format versions: generate shards in the new
-format, kill old search service, start new search service, delete old
-shards.
-
-
-Ranking
--------
-
-In absense of advanced signals (e.g. pagerank on symbol references),
-ranking options are limited: the following signals could be used for
-ranking
-
-   * number of atoms matched
-   * closeness to matches for other atoms
-   * quality of match: does match boundary coincide with a word boundary?
-   * file latest update time
-   * filename lengh
-   * tokenizer ranking: does a match fall comment or string literal?
-   * symbol ranking: it the match a symbol definition?
-
-For the latter, it is necessary to find symbol definitions and other
-sections within files on indexing. Several (imperfect) programs to do
-this already exist, eg. `ctags`.
-
-
-Query language
---------------
-
-Queries are stored as expression trees, using the following data
-structure:
-
-    Query:
-        Atom
-        | AND QueryList
-        | OR QueryList
-        | NOT Query
-        ;
-
-    Atom:
-        ConstQuery
-        | SubStringQuery
-        | RegexpQuery
-        | RepoQuery
-        | BranchQuery
-        ;
-
-Both SubStringQuery and RegexpQuery can apply to either file or
-contents, and can optionally be case-insensitive.
-
-ConstQuery (match everything, or match nothing) is a useful construct
-for partial evaluation of a query: for each index shard through which
-we search, we partially evaluate the query, eg. when the query is
-
-    and[substr:"needle" repo:"zoekt"]
-
-then we can rewrite the query to FALSE if we are looking at a shard
-for repository "bazel", skipping the entire shard.
-
-Each query must have at least one positive atom. Negations can only
-serve to prune results generated by positive atoms.
-
-
-Query parsing
--------------
-
-Strings in the input language are considered regular expressions
-but literal regular expressions are simplified to Substring queries,
-
-    a.*b => regexp:"a.*b"
-    a\.b => substring:"a.b"
-
-leading modifiers select different types of atoms, eg.
-
-    file:java => Substring_file:"java"
-    branch:master => Repo:"master"
-
-parentheses inside a string (possibly with escaped spaces) are
-interpreted as regular expressions, otherwise they are used for grouping
-
-    (abc def) => and[substring:"abc" substring:"def"]
-    (abc\ def) => regexp:"(abc def)"
-
-there is an implicit "AND" between elements of a parenthesized list.
-There is an "OR" operator, which has lower priority than the implicit
-"AND":
-
-    ppp qqq or rrr sss => or[and[substring:"ppp" substring:"qqq"] and[substring:"rrr" substring:"sss"]]
-
-
-GERRIT/GITILES INTEGRATION
-==========================
-
-Gerrit is a popular system for code review on open source
-projects. Its sister project Gitiles provides a browsing experience.
-
-Any code search integration with Gerrit should be made available in
-Gitiles.  Gerrit/Gitiles has a complex ACL system, so a codesearch
-solution for Gerrit/Gitiles should respect these ACLs.
-
-Since Gitiles knows the identity of the logged-in user, it can
-construct search queries that respect ACLs, and even filter results
-afterwards if necessary. In such a setup, only Gitiles is allowed to
-talk to the search service, so it should be protected from general
-access, e.g. by requiring authentication.
-
-A codesearch implementation for Gitiles would change Gitiles to show a
-search box on pages relating to a repository. When searching, Gitiles
-would also render the search results. The process is as follows:
-
-   * On receiving a query, Gitiles finds the list of branches visible to the user
-   * Gitiles sends the raw query, along with branches and repository to the search service
-   * The search service parses the query, and embeds it as follows
-
-    (AND original-query repo:REPO (OR "branch:visible-1" "branch:visible-2" .. ))
-
-   * The search service returns the search results, leaving it to
-     gitiles to render them. Gitiles can apply any further filtering
-     as necessary.
-
-
-SERVICE MANAGEMENT
-==================
-
-The above details how indexing and searching works. A fully fledged
-system also crawls repositories and (re)indexes them. Since the system
-is designed to run on a single machine, we provide a service
-management tool, with the following responsibilities:
-
-   * Poll git hosting sites (eg. github.com, googlesource.com), to fetch new updates
-   * Reindex any changed repositories
-   * Run the webserver; and restart if it goes down for any reason
-   * Delete old webserver logs
-
-
-Security
---------
-
-This section assumes that 'zoekt' is used as a public facing
-webserver, indexing publicly available data, serving on HTTPS without
-authentication.
-
-Since the UI is unauthenticated, there are no authentication secrets to steal.
-
-Since the code is public, there is no sensitive code to steal.
-
-This leaves us with the following senstive data:
-
-   * Credentials for accesssing git repostitories (eg. github access token)
-   * TLS server certificates
-   * Query logs
-
-The system handles the following untrusted data:
-
-   * code in git repositories
-   * search queries
-
-Since 'zoekt' itself is written in Go, it does not have memory
-security problems: at worst, a bug in the query parser would lead to a
-crash.
-
-The code to index is handled by `ctags` for symbol detection. The
-security risk this poses is mitigated by using a seccomp based
-sandboxing.
-
-
-Privacy
--------
-
-Webserver logs can contain privacy sensitive data (such as IP
-addresses and search queries). For this reason, the service management
-tool deletes them after a configurable period of time.
diff --git a/doc/faq.md b/doc/faq.md
deleted file mode 100644
index c8d81f0..0000000
--- a/doc/faq.md
+++ /dev/null
@@ -1,161 +0,0 @@
-# Frequently asked questions
-
-## Why codesearch?
-
-Software engineering is more about reading than writing code, and part
-of this process is finding the code that you should read. If you are
-working on a large project, then finding source code through
-navigation quickly becomes inefficient.
-
-Search engines let you find interesting code much faster than browsing
-code, in much the same way that search engines speed up finding things
-on the internet.
-
-## Can you give an example?
-
-I had to implement SSH hashed hostkey checking on a whim recently, and
-here is how I quickly zoomed into the relevant code using
-[our public zoekt instance](http://cs.bazel.build):
-
-* [hash host ssh](http://cs.bazel.build/search?q=hash+host+ssh&num=50): more than 20k results in 750 files, in 3 seconds
-
-* [hash host r:openssh](http://cs.bazel.build/search?q=hash+host+r%3Aopenssh&num=50): 6k results in 114 files, in 20ms
-
-* [hash host r:openssh known_host](http://cs.bazel.build/search?q=hash+host+r%3Aopenssh+known_host&num=50): 4k result in 42 files, in 13ms
-
-the last query still yielded a substantial number of results, but the
-function `hash_host` that I was looking for was the 3rd result from
-the first file.
-
-## What features make a code search engine great?
-
-Often, you don't know exactly what you are looking for, until you
-found it. Code search is effective because you can formulate an
-approximate query, and then refine it based on results you got. For
-this to work, you need the following features:
-
-* Coverage: the code that interests you should be available for searching
-
-* Speed: search should return useful results quickly (sub-second), so
-  you can iterate on queries
-
-* Approximate queries: matching should be done case insensitively, on
-  arbitrary substrings, so we don't have to know what we are looking
-  for in advance.
-
-* Filtering: we can winnow down results by composing more specific queries
-
-* Ranking: interesting results (eg. function definitions, whole word
-  matches) should be at the top.
-
-## How does `zoekt` provide for these?
-
-* Coverage: `zoekt` comes with tools to mirror parts of common Git
-  hosting sites. `cs.bazel.build` uses this to index most of the
-  Google authored open source software on github.com and
-  googlesource.com.
-
-* Speed: `zoekt` uses an index based on positional trigrams. For rare
-  strings, eg. `nienhuys`, this typically yields results in ~10ms if
-  the operating system caches are warm.
-
-* Approximate queries: `zoekt` supports substring patterns and regular
-  expressions, and can do case-insensitive matching on UTF-8 text.
-
-* Filtering: you can filter query by adding extra atoms (eg. `f:\.go$`
-  limits to Go source code), and filter out terms with `-`, so
-  `\blinus\b -torvalds` finds the Linuses other than Linus Torvalds.
-
-* Ranking: zoekt uses
-  [ctags](https://github.com/universal-ctags/ctags) to find
-  declarations, and these are boosted in the search ranking.
-
-
-## How does this compare to `grep -r`?
-
-Grep lets you find arbitrary substrings, but it doesn't scale to large
-corpuses, and lacks filtering and ranking.
-
-## What about my IDE?
-
-If your project fits into your IDE, than that is great.
-Unfortunately, loading projects into IDEs is slow, cumbersome, and not
-supported by all projects.
-
-## What about the search on `github.com`?
-
-Github's search has great coverage, but unfortunately, its search
-functionality doesn't support arbitrary substrings. For example, a
-query [for part of my
-surname](https://github.com/search?utf8=%E2%9C%93&q=nienhuy&type=Code)
-does not turn up anything (except this document), while
-[my complete
-name](https://github.com/search?utf8=%E2%9C%93&q=nienhuys&type=Code)
-does.
-
-## What about Etsy/Hound?
-
-[Etsy/hound](https://github.com/etsy/hound) is a code search engine
-which supports regular expressions over large corpuses, it is about
-10x slower than zoekt. However, there is only rudimentary support for
-filtering, and there is no symbol ranking.
-
-## What about livegrep?
-
-[livegrep](https://livegrep.com) is a code search engine which
-supports regular expressions over large corpuses. However, due to its
-indexing technique, it requires a lot of RAM and CPU.  There is only
-rudimentary support for filtering, and there is no symbol ranking.
-
-## How much resources does `zoekt` require?
-
-The search server should have local SSD to store the index file (which
-is 3.5x the corpus size), and have at least 20% more RAM than the
-corpus size.
-
-## Can I index multiple branches?
-
-Yes. You can index 64 branches (see also
-https://github.com/google/zoekt/issues/32). Files that are identical
-across branches take up space just once in the index.
-
-## How fast is the search?
-
-Rare strings, are extremely fast to retrieve, for example `r:torvalds
-crazy` (search "crazy" in the linux kernel) typically takes [about
-7-10ms on
-cs.bazel.build](http://cs.bazel.build/search?q=r%3Atorvalds+crazy&num=70).
-
-The speed for common strings is dominated by how many results you want
-to see. For example [r:torvalds license] can give some results
-quickly, but producing [all 86k
-results](http://cs.bazel.build/search?q=r%3Atorvalds+license&num=50000)
-takes between 100ms and 1 second. Then, streaming the results to your
-browser, and rendering the HTML takes several seconds.
-
-## How fast is the indexer?
-
-The Linux kernel (55K files, 545M data) takes about 160s to index on
-my x250 laptop using a single thread.  The process can be parallelized
-for speedup.
-
-## What does [cs.bazel.build](https://cs.bazel.build/) run on?
-
-Currently, it runs on a single Google Cloud VM with 16 vCPUs, 60G RAM and an
-attached physical SSD.
-
-## How does `zoekt` work?
-
-In short, it splits up the file in trigrams (groups of 3 unicode
-characters), and stores the offset of each occurrence. Substrings are
-found by searching different trigrams from the query at the correct
-distance apart.
-
-## I want to know more
-
-Some further background documentation
-
- * [Designdoc](design.md) for technical details
- * [Godoc](https://godoc.org/github.com/google/zoekt)
- * Gerrit 2016 user summit: [slides](https://storage.googleapis.com/gerrit-talks/summit/2016/zoekt.pdf)
- * Gerrit 2017 user summit: [transcript](https://gitenterprise.me/2017/11/01/gerrit-user-summit-zoekt-code-search-engine/),  [slides](https://storage.googleapis.com/gerrit-talks/summit/2017/Zoekt%20-%20improved%20codesearch.pdf), [video](https://www.youtube.com/watch?v=_-KTAvgJYdI)
diff --git a/doc/indexing.md b/doc/indexing.md
deleted file mode 100644
index c6563c1..0000000
--- a/doc/indexing.md
+++ /dev/null
@@ -1,15 +0,0 @@
-
-# Configuration parameters
-
-Parameters are in the `zoekt` section of the git-config.
-
-* `name`: name of the repository, typically HOST/PATH, eg. `github.com/hanwen/usb`.
-
-* `web-url`: base URL for linking to files, commits, and the repository, eg.
-`https://github.com/hanwen/usb`
-
-* `web-url-type`: type of URL, eg. github. Supported are cgit,
-  gitiles, gitweb and cgit.
-
-* `github-stars`, `github-forks`, `github-watchers`,
-  `github-subscribers`: counters for github interactions
diff --git a/eval.go b/eval.go
deleted file mode 100644
index de32847..0000000
--- a/eval.go
+++ /dev/null
@@ -1,504 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"regexp/syntax"
-	"sort"
-	"strings"
-
-	"golang.org/x/net/trace"
-
-	"github.com/google/zoekt/query"
-)
-
-const maxUInt16 = 0xffff
-
-// DebugScore controls whether we collect data on match scores are
-// constructed. Intended for use in tests.
-var DebugScore = false
-
-func (m *FileMatch) addScore(what string, s float64) {
-	if DebugScore {
-		m.Debug += fmt.Sprintf("%s:%f, ", what, s)
-	}
-	m.Score += s
-}
-
-func (d *indexData) simplify(in query.Q) query.Q {
-	eval := query.Map(in, func(q query.Q) query.Q {
-		if r, ok := q.(*query.Repo); ok {
-			return &query.Const{Value: strings.Contains(d.repoMetaData.Name, r.Pattern)}
-		}
-		if l, ok := q.(*query.Language); ok {
-			_, has := d.metaData.LanguageMap[l.Language]
-			if !has {
-				return &query.Const{Value: false}
-			}
-		}
-		return q
-	})
-	return query.Simplify(eval)
-}
-
-func (o *SearchOptions) SetDefaults() {
-	if o.ShardMaxMatchCount == 0 {
-		// We cap the total number of matches, so overly broad
-		// searches don't crash the machine.
-		o.ShardMaxMatchCount = 100000
-	}
-	if o.TotalMaxMatchCount == 0 {
-		o.TotalMaxMatchCount = 10 * o.ShardMaxMatchCount
-	}
-	if o.ShardMaxImportantMatch == 0 {
-		o.ShardMaxImportantMatch = 10
-	}
-	if o.TotalMaxImportantMatch == 0 {
-		o.TotalMaxImportantMatch = 10 * o.ShardMaxImportantMatch
-	}
-}
-
-func (d *indexData) Search(ctx context.Context, q query.Q, opts *SearchOptions) (sr *SearchResult, err error) {
-	copyOpts := *opts
-	opts = &copyOpts
-	opts.SetDefaults()
-	importantMatchCount := 0
-
-	var res SearchResult
-	if len(d.fileNameIndex) == 0 {
-		return &res, nil
-	}
-
-	select {
-	case <-ctx.Done():
-		res.Stats.ShardsSkipped++
-		return &res, nil
-	default:
-	}
-
-	tr := trace.New("indexData.Search", d.file.Name())
-	tr.LazyPrintf("opts: %+v", opts)
-	defer func() {
-		if sr != nil {
-			tr.LazyPrintf("num files: %d", len(sr.Files))
-			tr.LazyPrintf("stats: %+v", sr.Stats)
-		}
-		if err != nil {
-			tr.LazyPrintf("error: %v", err)
-			tr.SetError()
-		}
-		tr.Finish()
-	}()
-
-	q = d.simplify(q)
-	tr.LazyLog(q, true)
-	if c, ok := q.(*query.Const); ok && !c.Value {
-		return &res, nil
-	}
-
-	if opts.EstimateDocCount {
-		res.Stats.ShardFilesConsidered = len(d.fileBranchMasks)
-		return &res, nil
-	}
-
-	q = query.Map(q, query.ExpandFileContent)
-
-	mt, err := d.newMatchTree(q)
-	if err != nil {
-		return nil, err
-	}
-
-	totalAtomCount := 0
-	visitMatchTree(mt, func(t matchTree) {
-		totalAtomCount++
-	})
-
-	cp := &contentProvider{
-		id:    d,
-		stats: &res.Stats,
-	}
-
-	docCount := uint32(len(d.fileBranchMasks))
-	lastDoc := int(-1)
-
-nextFileMatch:
-	for {
-		canceled := false
-		select {
-		case <-ctx.Done():
-			canceled = true
-		default:
-		}
-
-		nextDoc := mt.nextDoc()
-		if int(nextDoc) <= lastDoc {
-			nextDoc = uint32(lastDoc + 1)
-		}
-		if nextDoc >= docCount {
-			break
-		}
-		lastDoc = int(nextDoc)
-
-		if canceled || (res.Stats.MatchCount >= opts.ShardMaxMatchCount && opts.ShardMaxMatchCount > 0) ||
-			(opts.ShardMaxImportantMatch > 0 && importantMatchCount >= opts.ShardMaxImportantMatch) {
-			res.Stats.FilesSkipped += d.repoListEntry.Stats.Documents - lastDoc
-			break
-		}
-
-		res.Stats.FilesConsidered++
-		mt.prepare(nextDoc)
-
-		cp.setDocument(nextDoc)
-
-		known := make(map[matchTree]bool)
-		for cost := costMin; cost <= costMax; cost++ {
-			v, ok := mt.matches(cp, cost, known)
-			if ok && !v {
-				continue nextFileMatch
-			}
-
-			if cost == costMax && !ok {
-				log.Panicf("did not decide. Repo %s, doc %d, known %v",
-					d.repoMetaData.Name, nextDoc, known)
-			}
-		}
-
-		fileMatch := FileMatch{
-			Repository: d.repoMetaData.Name,
-			FileName:   string(d.fileName(nextDoc)),
-			Checksum:   d.getChecksum(nextDoc),
-			Language:   d.languageMap[d.languages[nextDoc]],
-		}
-
-		if s := d.subRepos[nextDoc]; s > 0 {
-			if s >= uint32(len(d.subRepoPaths)) {
-				log.Panicf("corrupt index: subrepo %d beyond %v", s, d.subRepoPaths)
-			}
-			path := d.subRepoPaths[s]
-			fileMatch.SubRepositoryPath = path
-			sr := d.repoMetaData.SubRepoMap[path]
-			fileMatch.SubRepositoryName = sr.Name
-			if idx := d.branchIndex(nextDoc); idx >= 0 {
-				fileMatch.Version = sr.Branches[idx].Version
-			}
-		} else {
-			idx := d.branchIndex(nextDoc)
-			if idx >= 0 {
-				fileMatch.Version = d.repoMetaData.Branches[idx].Version
-			}
-		}
-
-		atomMatchCount := 0
-		visitMatches(mt, known, func(mt matchTree) {
-			atomMatchCount++
-		})
-		finalCands := gatherMatches(mt, known)
-
-		if len(finalCands) == 0 {
-			nm := d.fileName(nextDoc)
-			finalCands = append(finalCands,
-				&candidateMatch{
-					caseSensitive: false,
-					fileName:      true,
-					substrBytes:   nm,
-					substrLowered: nm,
-					file:          nextDoc,
-					runeOffset:    0,
-					byteOffset:    0,
-					byteMatchSz:   uint32(len(nm)),
-				})
-		}
-		fileMatch.LineMatches = cp.fillMatches(finalCands)
-
-		maxFileScore := 0.0
-		for i := range fileMatch.LineMatches {
-			if maxFileScore < fileMatch.LineMatches[i].Score {
-				maxFileScore = fileMatch.LineMatches[i].Score
-			}
-
-			// Order by ordering in file.
-			fileMatch.LineMatches[i].Score += scoreLineOrderFactor * (1.0 - (float64(i) / float64(len(fileMatch.LineMatches))))
-		}
-
-		// Maintain ordering of input files. This
-		// strictly dominates the in-file ordering of
-		// the matches.
-		fileMatch.addScore("fragment", maxFileScore)
-		fileMatch.addScore("atom", float64(atomMatchCount)/float64(totalAtomCount)*scoreFactorAtomMatch)
-
-		// Prefer earlier docs.
-		fileMatch.addScore("doc-order", scoreFileOrderFactor*(1.0-float64(nextDoc)/float64(len(d.boundaries))))
-		fileMatch.addScore("shard-order", scoreShardRankFactor*float64(d.repoMetaData.Rank)/maxUInt16)
-
-		if fileMatch.Score > scoreImportantThreshold {
-			importantMatchCount++
-		}
-		fileMatch.Branches = d.gatherBranches(nextDoc, mt, known)
-		sortMatchesByScore(fileMatch.LineMatches)
-		if opts.Whole {
-			fileMatch.Content = cp.data(false)
-		}
-
-		res.Files = append(res.Files, fileMatch)
-		res.Stats.MatchCount += len(fileMatch.LineMatches)
-		res.Stats.FileCount++
-	}
-	SortFilesByScore(res.Files)
-
-	addRepo(&res, &d.repoMetaData)
-	for _, v := range d.repoMetaData.SubRepoMap {
-		addRepo(&res, v)
-	}
-
-	visitMatchTree(mt, func(mt matchTree) {
-		if atom, ok := mt.(interface{ updateStats(*Stats) }); ok {
-			atom.updateStats(&res.Stats)
-		}
-	})
-	return &res, nil
-}
-
-func addRepo(res *SearchResult, repo *Repository) {
-	if res.RepoURLs == nil {
-		res.RepoURLs = map[string]string{}
-	}
-	res.RepoURLs[repo.Name] = repo.FileURLTemplate
-
-	if res.LineFragments == nil {
-		res.LineFragments = map[string]string{}
-	}
-	res.LineFragments[repo.Name] = repo.LineFragmentTemplate
-}
-
-type sortByOffsetSlice []*candidateMatch
-
-func (m sortByOffsetSlice) Len() int      { return len(m) }
-func (m sortByOffsetSlice) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
-func (m sortByOffsetSlice) Less(i, j int) bool {
-	return m[i].byteOffset < m[j].byteOffset
-}
-
-// Gather matches from this document. This never returns a mixture of
-// filename/content matches: if there are content matches, all
-// filename matches are trimmed from the result. The matches are
-// returned in document order and are non-overlapping.
-func gatherMatches(mt matchTree, known map[matchTree]bool) []*candidateMatch {
-	var cands []*candidateMatch
-	visitMatches(mt, known, func(mt matchTree) {
-		if smt, ok := mt.(*substrMatchTree); ok {
-			cands = append(cands, smt.current...)
-		}
-		if rmt, ok := mt.(*regexpMatchTree); ok {
-			cands = append(cands, rmt.found...)
-		}
-	})
-
-	foundContentMatch := false
-	for _, c := range cands {
-		if !c.fileName {
-			foundContentMatch = true
-			break
-		}
-	}
-
-	res := cands[:0]
-	for _, c := range cands {
-		if !foundContentMatch || !c.fileName {
-			res = append(res, c)
-		}
-	}
-	cands = res
-
-	// Merge adjacent candidates. This guarantees that the matches
-	// are non-overlapping.
-	sort.Sort((sortByOffsetSlice)(cands))
-	res = cands[:0]
-	for i, c := range cands {
-		if i == 0 {
-			res = append(res, c)
-			continue
-		}
-		last := res[len(res)-1]
-		lastEnd := last.byteOffset + last.byteMatchSz
-		end := c.byteOffset + c.byteMatchSz
-		if lastEnd >= c.byteOffset {
-			if end > lastEnd {
-				last.byteMatchSz = end - last.byteOffset
-			}
-			continue
-		}
-
-		res = append(res, c)
-	}
-
-	return res
-}
-
-func (d *indexData) branchIndex(docID uint32) int {
-	mask := d.fileBranchMasks[docID]
-	idx := 0
-	for mask != 0 {
-		if mask&0x1 != 0 {
-			return idx
-		}
-		idx++
-		mask >>= 1
-	}
-	return -1
-}
-
-// gatherBranches returns a list of branch names.
-func (d *indexData) gatherBranches(docID uint32, mt matchTree, known map[matchTree]bool) []string {
-	foundBranchQuery := false
-	var branches []string
-
-	visitMatches(mt, known, func(mt matchTree) {
-		bq, ok := mt.(*branchQueryMatchTree)
-		if ok {
-			foundBranchQuery = true
-			branches = append(branches,
-				d.branchNames[uint(bq.mask)])
-		}
-	})
-
-	if !foundBranchQuery {
-		mask := d.fileBranchMasks[docID]
-		id := uint32(1)
-		for mask != 0 {
-			if mask&0x1 != 0 {
-				branches = append(branches, d.branchNames[uint(id)])
-			}
-			id <<= 1
-			mask >>= 1
-		}
-	}
-	return branches
-}
-
-func (d *indexData) List(ctx context.Context, q query.Q) (rl *RepoList, err error) {
-	tr := trace.New("indexData.List", d.file.Name())
-	defer func() {
-		if rl != nil {
-			tr.LazyPrintf("repos size: %d", len(rl.Repos))
-			tr.LazyPrintf("crashes: %d", rl.Crashes)
-		}
-		if err != nil {
-			tr.LazyPrintf("error: %v", err)
-			tr.SetError()
-		}
-		tr.Finish()
-	}()
-
-	q = d.simplify(q)
-	tr.LazyLog(q, true)
-	c, ok := q.(*query.Const)
-
-	if !ok {
-		return nil, fmt.Errorf("List should receive Repo-only query")
-	}
-
-	l := &RepoList{}
-	if c.Value {
-		l.Repos = append(l.Repos, &d.repoListEntry)
-	}
-	return l, nil
-}
-
-// regexpToMatchTreeRecursive converts a regular expression to a matchTree mt. If
-// mt is equivalent to the input r, isEqual = true and the matchTree can be used
-// in place of the regex r. If singleLine = true, then the matchTree and all
-// its children only match terms on the same line. singleLine is used during
-// recursion to decide whether to return an andLineMatchTree (singleLine = true)
-// or a andMatchTree (singleLine = false).
-func (d *indexData) regexpToMatchTreeRecursive(r *syntax.Regexp, minTextSize int, fileName bool, caseSensitive bool) (mt matchTree, isEqual bool, singleLine bool, err error) {
-	// TODO - we could perhaps transform Begin/EndText in '\n'?
-	// TODO - we could perhaps transform CharClass in (OrQuery )
-	// if there are just a few runes, and part of a OpConcat?
-	switch r.Op {
-	case syntax.OpLiteral:
-		s := string(r.Rune)
-		if len(s) >= minTextSize {
-			mt, err := d.newSubstringMatchTree(&query.Substring{Pattern: s, FileName: fileName, CaseSensitive: caseSensitive})
-			return mt, true, !strings.Contains(s, "\n"), err
-		}
-	case syntax.OpCapture:
-		return d.regexpToMatchTreeRecursive(r.Sub[0], minTextSize, fileName, caseSensitive)
-
-	case syntax.OpPlus:
-		return d.regexpToMatchTreeRecursive(r.Sub[0], minTextSize, fileName, caseSensitive)
-
-	case syntax.OpRepeat:
-		if r.Min == 1 {
-			return d.regexpToMatchTreeRecursive(r.Sub[0], minTextSize, fileName, caseSensitive)
-		} else if r.Min > 1 {
-			// (x){2,} can't be expressed precisely by the matchTree
-			mt, _, singleLine, err := d.regexpToMatchTreeRecursive(r.Sub[0], minTextSize, fileName, caseSensitive)
-			return mt, false, singleLine, err
-		}
-	case syntax.OpConcat, syntax.OpAlternate:
-		var qs []matchTree
-		isEq := true
-		singleLine = true
-		for _, sr := range r.Sub {
-			if sq, subIsEq, subSingleLine, err := d.regexpToMatchTreeRecursive(sr, minTextSize, fileName, caseSensitive); sq != nil {
-				if err != nil {
-					return nil, false, false, err
-				}
-				isEq = isEq && subIsEq
-				singleLine = singleLine && subSingleLine
-				qs = append(qs, sq)
-			}
-		}
-		if r.Op == syntax.OpConcat {
-			if len(qs) > 1 {
-				isEq = false
-			}
-			newQs := make([]matchTree, 0, len(qs))
-			for _, q := range qs {
-				if _, ok := q.(*bruteForceMatchTree); ok {
-					continue
-				}
-				newQs = append(newQs, q)
-			}
-			if len(newQs) == 1 {
-				return newQs[0], isEq, singleLine, nil
-			}
-			if len(newQs) == 0 {
-				return &bruteForceMatchTree{}, isEq, singleLine, nil
-			}
-			if singleLine {
-				return &andLineMatchTree{andMatchTree{children: newQs}}, isEq, singleLine, nil
-			}
-			return &andMatchTree{newQs}, isEq, singleLine, nil
-		}
-		for _, q := range qs {
-			if _, ok := q.(*bruteForceMatchTree); ok {
-				return q, isEq, false, nil
-			}
-		}
-		if len(qs) == 0 {
-			return &noMatchTree{"const"}, isEq, false, nil
-		}
-		return &orMatchTree{qs}, isEq, false, nil
-	case syntax.OpStar:
-		if r.Sub[0].Op == syntax.OpAnyCharNotNL {
-			return &bruteForceMatchTree{}, false, true, nil
-		}
-	}
-	return &bruteForceMatchTree{}, false, false, nil
-}
diff --git a/eval_test.go b/eval_test.go
deleted file mode 100644
index ce16a60..0000000
--- a/eval_test.go
+++ /dev/null
@@ -1,128 +0,0 @@
-// Copyright 2020 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"reflect"
-	"regexp/syntax"
-	"strings"
-	"testing"
-
-	"github.com/google/zoekt/query"
-)
-
-var opnames = map[syntax.Op]string{
-	syntax.OpNoMatch:        "OpNoMatch",
-	syntax.OpEmptyMatch:     "OpEmptyMatch",
-	syntax.OpLiteral:        "OpLiteral",
-	syntax.OpCharClass:      "OpCharClass",
-	syntax.OpAnyCharNotNL:   "OpAnyCharNotNL",
-	syntax.OpAnyChar:        "OpAnyChar",
-	syntax.OpBeginLine:      "OpBeginLine",
-	syntax.OpEndLine:        "OpEndLine",
-	syntax.OpBeginText:      "OpBeginText",
-	syntax.OpEndText:        "OpEndText",
-	syntax.OpWordBoundary:   "OpWordBoundary",
-	syntax.OpNoWordBoundary: "OpNoWordBoundary",
-	syntax.OpCapture:        "OpCapture",
-	syntax.OpStar:           "OpStar",
-	syntax.OpPlus:           "OpPlus",
-	syntax.OpQuest:          "OpQuest",
-	syntax.OpRepeat:         "OpRepeat",
-	syntax.OpConcat:         "OpConcat",
-	syntax.OpAlternate:      "OpAlternate",
-}
-
-func printRegexp(t *testing.T, r *syntax.Regexp, lvl int) {
-	t.Logf("%s%s ch: %d", strings.Repeat(" ", lvl), opnames[r.Op], len(r.Sub))
-	for _, s := range r.Sub {
-		printRegexp(t, s, lvl+1)
-	}
-}
-
-func substrMT(pattern string) matchTree {
-	d := &indexData{}
-	mt, _ := d.newSubstringMatchTree(&query.Substring{
-		Pattern: pattern,
-	})
-	return mt
-}
-
-func TestRegexpParse(t *testing.T) {
-	type testcase struct {
-		in           string
-		query        matchTree
-		isEquivalent bool
-	}
-
-	cases := []testcase{
-		{"(foo|)bar", substrMT("bar"), false},
-		{"(foo|)", &bruteForceMatchTree{}, false},
-		{"(foo|bar)baz.*bla", &andMatchTree{[]matchTree{
-			&orMatchTree{[]matchTree{
-				substrMT("foo"),
-				substrMT("bar"),
-			}},
-			substrMT("baz"),
-			substrMT("bla"),
-		}}, false},
-		{
-			"^[a-z](People)+barrabas$",
-			&andMatchTree{[]matchTree{
-				substrMT("People"),
-				substrMT("barrabas"),
-			}}, false,
-		},
-		{"foo", substrMT("foo"), true},
-		{"^foo", substrMT("foo"), false},
-		{"(foo) (bar)", &andMatchTree{[]matchTree{substrMT("foo"), substrMT("bar")}}, false},
-		{"(thread|needle|haystack)", &orMatchTree{[]matchTree{
-			substrMT("thread"),
-			substrMT("needle"),
-			substrMT("haystack"),
-		}}, true},
-		{"(foo)(?-s:.)*?(bar)", &andLineMatchTree{andMatchTree{[]matchTree{
-			substrMT("foo"),
-			substrMT("bar"),
-		}}}, false},
-		{"(foo)(?-s:.)*?[[:space:]](?-s:.)*?(bar)", &andMatchTree{[]matchTree{
-			substrMT("foo"),
-			substrMT("bar"),
-		}}, false},
-		{"(foo){2,}", substrMT("foo"), false},
-		{"(...)(...)", &bruteForceMatchTree{}, false},
-	}
-
-	for _, c := range cases {
-		r, err := syntax.Parse(c.in, syntax.Perl)
-		if err != nil {
-			t.Errorf("Parse(%q): %v", c.in, err)
-			continue
-		}
-		d := indexData{}
-		q := query.Regexp{
-			Regexp: r,
-		}
-		gotQuery, isEq, _, _ := d.regexpToMatchTreeRecursive(q.Regexp, 3, q.FileName, q.CaseSensitive)
-		if !reflect.DeepEqual(c.query, gotQuery) {
-			printRegexp(t, r, 0)
-			t.Errorf("regexpToQuery(%q): got %v, want %v", c.in, gotQuery, c.query)
-		}
-		if isEq != c.isEquivalent {
-			printRegexp(t, r, 0)
-			t.Errorf("regexpToQuery(%q): got %v, want %v", c.in, isEq, c.isEquivalent)
-		}
-	}
-}
diff --git a/gitindex/clone.go b/gitindex/clone.go
deleted file mode 100644
index 11b2c87..0000000
--- a/gitindex/clone.go
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"bytes"
-	"log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"sort"
-
-	git "github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/config"
-)
-
-// CloneRepo clones one repository, adding the given config
-// settings. It returns the bare repo directory. The `name` argument
-// determines where the repo is stored relative to `destDir`. Returns
-// the directory of the repository.
-func CloneRepo(destDir, name, cloneURL string, settings map[string]string) (string, error) {
-	parent := filepath.Join(destDir, filepath.Dir(name))
-	if err := os.MkdirAll(parent, 0o755); err != nil {
-		return "", err
-	}
-
-	repoDest := filepath.Join(parent, filepath.Base(name)+".git")
-	if _, err := os.Lstat(repoDest); err == nil {
-		return "", nil
-	}
-
-	var keys []string
-	for k := range settings {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	var config []string
-	for _, k := range keys {
-		if settings[k] != "" {
-			config = append(config, "--config", k+"="+settings[k])
-		}
-	}
-
-	cmd := exec.Command(
-		"git", "clone", "--bare", "--verbose", "--progress",
-	)
-	cmd.Args = append(cmd.Args, config...)
-	cmd.Args = append(cmd.Args, cloneURL, repoDest)
-
-	// Prevent prompting
-	cmd.Stdin = &bytes.Buffer{}
-	log.Println("running:", cmd.Args)
-	if err := cmd.Run(); err != nil {
-		return "", err
-	}
-
-	if err := setFetch(repoDest, "origin", "+refs/heads/*:refs/heads/*"); err != nil {
-		log.Printf("addFetch: %v", err)
-	}
-	return repoDest, nil
-}
-
-func setFetch(repoDir, remote, refspec string) error {
-	repo, err := git.PlainOpen(repoDir)
-	if err != nil {
-		return err
-	}
-
-	cfg, err := repo.Config()
-	if err != nil {
-		return err
-	}
-
-	rm := cfg.Remotes[remote]
-	if rm != nil {
-		rm.Fetch = []config.RefSpec{config.RefSpec(refspec)}
-	}
-	if err := repo.Storer.SetConfig(cfg); err != nil {
-		return err
-	}
-
-	return nil
-}
diff --git a/gitindex/clone_test.go b/gitindex/clone_test.go
deleted file mode 100644
index 8445416..0000000
--- a/gitindex/clone_test.go
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2019 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"testing"
-
-	git "github.com/go-git/go-git/v5"
-)
-
-func TestSetRemote(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-	script := `mkdir orig
-cd orig
-git init
-cd ..
-git clone orig/.git clone.git
-`
-
-	cmd := exec.Command("/bin/sh", "-euxc", script)
-	cmd.Dir = dir
-
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("execution error: %v, output %s", err, out)
-	}
-
-	r := dir + "/clone.git"
-	if err := setFetch(r, "origin", "+refs/heads/*:refs/heads/*"); err != nil {
-		t.Fatalf("addFetch: %v", err)
-	}
-
-	repo, err := git.PlainOpen(r)
-	if err != nil {
-		t.Fatal("PlainOpen", err)
-	}
-
-	rm, err := repo.Remote("origin")
-	if err != nil {
-		t.Fatal("Remote", err)
-	}
-	if got, want := rm.Config().Fetch[0].String(), "+refs/heads/*:refs/heads/*"; got != want {
-		t.Fatalf("got %q want %q", got, want)
-	}
-}
diff --git a/gitindex/delete.go b/gitindex/delete.go
deleted file mode 100644
index f53e1af..0000000
--- a/gitindex/delete.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package gitindex
-
-import (
-	"fmt"
-	"log"
-	"net/url"
-	"os"
-	"path/filepath"
-)
-
-// DeleteRepos deletes stale repos under a specific path in disk. The `names`
-// argument stores names of repos retrieved from the git hosting site
-// and is used along with the `filter` argument to decide on repo deletion.
-func DeleteRepos(baseDir string, urlPrefix *url.URL, names map[string]struct{}, filter *Filter) error {
-	paths, err := ListRepos(baseDir, urlPrefix)
-	if err != nil {
-		return err
-	}
-	var toDelete []string
-	for _, p := range paths {
-		_, exists := names[p]
-		if filter.Include(filepath.Base(p)) && !exists {
-			toDelete = append(toDelete, p)
-		}
-	}
-
-	if len(toDelete) > 0 {
-		log.Printf("deleting repos %v", toDelete)
-	}
-
-	var errs []string
-	for _, d := range toDelete {
-		if err := os.RemoveAll(filepath.Join(baseDir, d)); err != nil {
-			errs = append(errs, err.Error())
-		}
-	}
-	if len(errs) > 0 {
-		return fmt.Errorf("errors: %v", errs)
-	}
-	return nil
-}
diff --git a/gitindex/delete_test.go b/gitindex/delete_test.go
deleted file mode 100644
index 381e6e0..0000000
--- a/gitindex/delete_test.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package gitindex
-
-import (
-	"io/ioutil"
-	"net/url"
-	"os"
-	"path/filepath"
-	"reflect"
-	"testing"
-)
-
-func TestDeleteRepos(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	if err := createSubmoduleRepo(dir); err != nil {
-		t.Error("createSubmoduleRepo", err)
-	}
-
-	reposBefore, err := FindGitRepos(dir)
-	if err != nil {
-		t.Error("FindGitRepos", err)
-	}
-
-	gotBefore := map[string]struct{}{}
-	for _, r := range reposBefore {
-		p, err := filepath.Rel(dir, r)
-		if err != nil {
-			t.Fatalf("Relative: %v", err)
-		}
-
-		gotBefore[p] = struct{}{}
-	}
-
-	wantBefore := map[string]struct{}{
-		"gerrit.googlesource.com/bdir.git":     {},
-		"gerrit.googlesource.com/sub/bdir.git": {},
-		"adir/.git":                            {},
-		"bdir/.git":                            {},
-		"gerrit.googlesource.com/adir.git":     {},
-	}
-
-	if !reflect.DeepEqual(gotBefore, wantBefore) {
-		t.Fatalf("got %v want %v", gotBefore, wantBefore)
-	}
-
-	aURL, _ := url.Parse("http://gerrit.googlesource.com")
-	aURL.Path = "sub"
-	names := map[string]struct{}{
-		"bdir/.git":                        {},
-		"gerrit.googlesource.com/adir.git": {},
-	}
-	filter, _ := NewFilter("", "")
-
-	err = DeleteRepos(dir, aURL, names, filter)
-	if err != nil {
-		t.Fatalf("DeleteRepos: %T", err)
-	}
-	reposAfter, err := FindGitRepos(dir)
-	if err != nil {
-		t.Error("FindGitRepos", err)
-	}
-
-	gotAfter := map[string]struct{}{}
-	for _, r := range reposAfter {
-		p, err := filepath.Rel(dir, r)
-		if err != nil {
-			t.Fatalf("Relative: %v", err)
-		}
-
-		gotAfter[p] = struct{}{}
-	}
-	wantAfter := map[string]struct{}{
-		"gerrit.googlesource.com/bdir.git": {},
-		"adir/.git":                        {},
-		"bdir/.git":                        {},
-		"gerrit.googlesource.com/adir.git": {},
-	}
-
-	if !reflect.DeepEqual(gotAfter, wantAfter) {
-		t.Errorf("got %v want %v", gotAfter, wantAfter)
-	}
-}
diff --git a/gitindex/filter.go b/gitindex/filter.go
deleted file mode 100644
index eba82f9..0000000
--- a/gitindex/filter.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import "regexp"
-
-// Filter is a include/exclude filter to be used for repo names.
-type Filter struct {
-	inc, exc *regexp.Regexp
-}
-
-// Include returns true if the name passes the filter.
-func (f *Filter) Include(name string) bool {
-	if f.inc != nil {
-		if !f.inc.MatchString(name) {
-			return false
-		}
-	}
-	if f.exc != nil {
-		if f.exc.MatchString(name) {
-			return false
-		}
-	}
-	return true
-}
-
-// NewFilter creates a new filter.
-func NewFilter(includeRegex, excludeRegex string) (*Filter, error) {
-	f := &Filter{}
-	var err error
-	if includeRegex != "" {
-		f.inc, err = regexp.Compile(includeRegex)
-
-		if err != nil {
-			return nil, err
-		}
-	}
-	if excludeRegex != "" {
-		f.exc, err = regexp.Compile(excludeRegex)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return f, nil
-}
diff --git a/gitindex/index.go b/gitindex/index.go
deleted file mode 100644
index 5715406..0000000
--- a/gitindex/index.go
+++ /dev/null
@@ -1,521 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package gitindex provides functions for indexing Git repositories.
-package gitindex
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"log"
-	"math"
-	"net/url"
-	"os"
-	"path/filepath"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-
-	"github.com/go-git/go-git/v5/config"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
-
-	git "github.com/go-git/go-git/v5"
-)
-
-// RepoModTime returns the time of last fetch of a git repository.
-func RepoModTime(dir string) (time.Time, error) {
-	var last time.Time
-	refDir := filepath.Join(dir, "refs")
-	if _, err := os.Lstat(refDir); err == nil {
-		if err := filepath.Walk(refDir,
-			func(_ string, fi os.FileInfo, _ error) error {
-				if !fi.IsDir() && last.Before(fi.ModTime()) {
-					last = fi.ModTime()
-				}
-				return nil
-			}); err != nil {
-			return last, err
-		}
-	}
-
-	// git gc compresses refs into the following file:
-	for _, fn := range []string{"info/refs", "packed-refs"} {
-		if fi, err := os.Lstat(filepath.Join(dir, fn)); err == nil && !fi.IsDir() && last.Before(fi.ModTime()) {
-			last = fi.ModTime()
-		}
-	}
-
-	return last, nil
-}
-
-// FindGitRepos finds directories holding git repositories below the
-// given directory. It will find both bare and the ".git" dirs in
-// non-bare repositories. It returns the full path including the dir
-// passed in.
-func FindGitRepos(dir string) ([]string, error) {
-	arg, err := filepath.Abs(dir)
-	if err != nil {
-		return nil, err
-	}
-	var dirs []string
-	if err := filepath.Walk(arg, func(name string, fi os.FileInfo, err error) error {
-		// Best-effort, ignore filepath.Walk failing
-		if err != nil {
-			return nil
-		}
-
-		if fi, err := os.Lstat(filepath.Join(name, ".git")); err == nil && fi.IsDir() {
-			dirs = append(dirs, filepath.Join(name, ".git"))
-			return filepath.SkipDir
-		}
-
-		if !strings.HasSuffix(name, ".git") || !fi.IsDir() {
-			return nil
-		}
-
-		fi, err = os.Lstat(filepath.Join(name, "objects"))
-		if err != nil || !fi.IsDir() {
-			return nil
-		}
-
-		dirs = append(dirs, name)
-		return filepath.SkipDir
-	}); err != nil {
-		return nil, err
-	}
-
-	return dirs, nil
-}
-
-// setTemplates fills in URL templates for known git hosting
-// sites.
-func setTemplates(repo *zoekt.Repository, u *url.URL, typ string) error {
-	repo.URL = u.String()
-	switch typ {
-	case "gitiles":
-		/// eg. https://gerrit.googlesource.com/gitiles/+/master/tools/run_dev.sh#20
-		repo.CommitURLTemplate = u.String() + "/+/{{.Version}}"
-		repo.FileURLTemplate = u.String() + "/+/{{.Version}}/{{.Path}}"
-		repo.LineFragmentTemplate = "#{{.LineNumber}}"
-	case "github":
-		// eg. https://github.com/hanwen/go-fuse/blob/notify/genversion.sh#L10
-		repo.CommitURLTemplate = u.String() + "/commit/{{.Version}}"
-		repo.FileURLTemplate = u.String() + "/blob/{{.Version}}/{{.Path}}"
-		repo.LineFragmentTemplate = "#L{{.LineNumber}}"
-	case "cgit":
-		// http://git.savannah.gnu.org/cgit/lilypond.git/tree/elisp/lilypond-mode.el?h=dev/philh&id=b2ca0fefe3018477aaca23b6f672c7199ba5238e#n100
-		repo.CommitURLTemplate = u.String() + "/commit/?id={{.Version}}"
-		repo.FileURLTemplate = u.String() + "/tree/{{.Path}}/?id={{.Version}}"
-		repo.LineFragmentTemplate = "#n{{.LineNumber}}"
-	case "gitweb":
-		// https://gerrit.libreoffice.org/gitweb?p=online.git;a=blob;f=Makefile.am;h=cfcfd7c36fbae10e269653dc57a9b68c92d4c10b;hb=848145503bf7b98ce4a4aa0a858a0d71dd0dbb26#l10
-		repo.FileURLTemplate = u.String() + ";a=blob;f={{.Path}};hb={{.Version}}"
-		repo.CommitURLTemplate = u.String() + ";a=commit;h={{.Version}}"
-		repo.LineFragmentTemplate = "#l{{.LineNumber}}"
-	case "source.bazel.build":
-		// https://source.bazel.build/bazel/+/57bc201346e61c62a921c1cbf32ad24f185c10c9
-		// https://source.bazel.build/bazel/+/57bc201346e61c62a921c1cbf32ad24f185c10c9:tools/cpp/BUILD.empty;l=10
-		repo.CommitURLTemplate = u.String() + "/+/{{.Version}}"
-		repo.FileURLTemplate = u.String() + "/+/{{.Version}}:{{.Path}}"
-		repo.LineFragmentTemplate = ";l={{.LineNumber}}"
-	case "bitbucket-server":
-		// https://<bitbucketserver-host>/projects/<project>/repos/<repo>/commits/5be7ca73b898bf17a08e607918accfdeafe1e0bc
-		// https://<bitbucketserver-host>/projects/<project>/repos/<repo>/browse/<file>?at=5be7ca73b898bf17a08e607918accfdeafe1e0bc
-		repo.CommitURLTemplate = u.String() + "/commits/{{.Version}}"
-		repo.FileURLTemplate = u.String() + "/{{.Path}}?at={{.Version}}"
-		repo.LineFragmentTemplate = "#{{.LineNumber}}"
-	case "gitlab":
-		repo.CommitURLTemplate = u.String() + "/commit/{{.Version}}"
-		repo.FileURLTemplate = u.String() + "/blob/{{.Version}}/{{.Path}}"
-		repo.LineFragmentTemplate = "#L{{.LineNumber}}"
-	default:
-		return fmt.Errorf("URL scheme type %q unknown", typ)
-	}
-	return nil
-}
-
-// getCommit returns a tree object for the given reference.
-func getCommit(repo *git.Repository, prefix, ref string) (*object.Commit, error) {
-	sha1, err := repo.ResolveRevision(plumbing.Revision(ref))
-	// ref might be a branch name (e.g. "master") add branch prefix and try again.
-	if err != nil {
-		sha1, err = repo.ResolveRevision(plumbing.Revision(filepath.Join(prefix, ref)))
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	commitObj, err := repo.CommitObject(*sha1)
-	if err != nil {
-		return nil, err
-	}
-	return commitObj, nil
-}
-
-func configLookupRemoteURL(cfg *config.Config, key string) string {
-	rc := cfg.Remotes[key]
-	if rc == nil || len(rc.URLs) == 0 {
-		return ""
-	}
-	return rc.URLs[0]
-}
-
-func setTemplatesFromConfig(desc *zoekt.Repository, repoDir string) error {
-	repo, err := git.PlainOpen(repoDir)
-	if err != nil {
-		return err
-	}
-
-	cfg, err := repo.Config()
-	if err != nil {
-		return err
-	}
-
-	sec := cfg.Raw.Section("zoekt")
-
-	webURLStr := sec.Options.Get("web-url")
-	webURLType := sec.Options.Get("web-url-type")
-
-	if webURLType != "" && webURLStr != "" {
-		webURL, err := url.Parse(webURLStr)
-		if err != nil {
-			return err
-		}
-		if err := setTemplates(desc, webURL, webURLType); err != nil {
-			return err
-		}
-	}
-
-	name := sec.Options.Get("name")
-	if name != "" {
-		desc.Name = name
-	} else {
-		remoteURL := configLookupRemoteURL(cfg, "origin")
-		if remoteURL == "" {
-			return nil
-		}
-		u, err := url.Parse(remoteURL)
-		if err != nil {
-			return err
-		}
-		if err := SetTemplatesFromOrigin(desc, u); err != nil {
-			return err
-		}
-	}
-
-	if desc.RawConfig == nil {
-		desc.RawConfig = map[string]string{}
-	}
-	for _, o := range sec.Options {
-		desc.RawConfig[o.Key] = o.Value
-	}
-
-	// Ranking info.
-
-	// Github:
-	traction := 0
-	for _, s := range []string{"github-stars", "github-forks", "github-watchers", "github-subscribers"} {
-		f, err := strconv.Atoi(sec.Options.Get(s))
-		if err == nil {
-			traction += f
-		}
-	}
-
-	if strings.Contains(desc.Name, "googlesource.com/") && traction == 0 {
-		// Pretend everything on googlesource.com has 1000
-		// github stars.
-		traction = 1000
-	}
-
-	if traction > 0 {
-		l := math.Log(float64(traction))
-		desc.Rank = uint16((1.0 - 1.0/math.Pow(1+l, 0.6)) * 10000)
-	}
-
-	return nil
-}
-
-// SetTemplatesFromOrigin fills in templates based on the origin URL.
-func SetTemplatesFromOrigin(desc *zoekt.Repository, u *url.URL) error {
-	desc.Name = filepath.Join(u.Host, strings.TrimSuffix(u.Path, ".git"))
-
-	if strings.HasSuffix(u.Host, ".googlesource.com") {
-		return setTemplates(desc, u, "gitiles")
-	} else if u.Host == "github.com" {
-		u.Path = strings.TrimSuffix(u.Path, ".git")
-		return setTemplates(desc, u, "github")
-	} else {
-		return fmt.Errorf("unknown git hosting site %q", u)
-	}
-}
-
-// The Options structs controls details of the indexing process.
-type Options struct {
-	// The repository to be indexed.
-	RepoDir string
-
-	// If set, follow submodule links. This requires RepoCacheDir to be set.
-	Submodules bool
-
-	// If set, skip indexing if the existing index shard is newer
-	// than the refs in the repository.
-	Incremental bool
-
-	// Don't error out if some branch is missing
-	AllowMissingBranch bool
-
-	// Specifies the root of a Repository cache. Needed for submodule indexing.
-	RepoCacheDir string
-
-	// Indexing options.
-	BuildOptions build.Options
-
-	// Prefix of the branch to index, e.g. `remotes/origin`.
-	BranchPrefix string
-
-	// List of branch names to index, e.g. []string{"HEAD", "stable"}
-	Branches []string
-}
-
-func expandBranches(repo *git.Repository, bs []string, prefix string) ([]string, error) {
-	var result []string
-	for _, b := range bs {
-		if b == "HEAD" {
-			ref, err := repo.Head()
-			if err != nil {
-				return nil, err
-			}
-
-			result = append(result, strings.TrimPrefix(ref.Name().String(), prefix))
-			continue
-		}
-
-		if strings.Contains(b, "*") {
-			iter, err := repo.Branches()
-			if err != nil {
-				return nil, err
-			}
-
-			defer iter.Close()
-			for {
-				ref, err := iter.Next()
-				if err == io.EOF {
-					break
-				}
-				if err != nil {
-					return nil, err
-				}
-
-				name := ref.Name().Short()
-				if matched, err := filepath.Match(b, name); err != nil {
-					return nil, err
-				} else if !matched {
-					continue
-				}
-
-				result = append(result, strings.TrimPrefix(name, prefix))
-			}
-			continue
-		}
-
-		result = append(result, b)
-	}
-
-	return result, nil
-}
-
-// IndexGitRepo indexes the git repository as specified by the options.
-func IndexGitRepo(opts Options) error {
-	// Set max thresholds, since we use them in this function.
-	opts.BuildOptions.SetDefaults()
-	if opts.RepoDir == "" {
-		return fmt.Errorf("gitindex: must set RepoDir")
-	}
-
-	opts.BuildOptions.RepositoryDescription.Source = opts.RepoDir
-	repo, err := git.PlainOpen(opts.RepoDir)
-	if err != nil {
-		return err
-	}
-
-	if err := setTemplatesFromConfig(&opts.BuildOptions.RepositoryDescription, opts.RepoDir); err != nil {
-		log.Printf("setTemplatesFromConfig(%s): %s", opts.RepoDir, err)
-	}
-
-	repoCache := NewRepoCache(opts.RepoCacheDir)
-
-	// branch => (path, sha1) => repo.
-	repos := map[fileKey]BlobLocation{}
-
-	// fileKey => branches
-	branchMap := map[fileKey][]string{}
-
-	// Branch => Repo => SHA1
-	branchVersions := map[string]map[string]plumbing.Hash{}
-
-	branches, err := expandBranches(repo, opts.Branches, opts.BranchPrefix)
-	if err != nil {
-		return err
-	}
-	for _, b := range branches {
-		commit, err := getCommit(repo, opts.BranchPrefix, b)
-		if err != nil {
-			if opts.AllowMissingBranch && err.Error() == "reference not found" {
-				continue
-			}
-
-			return err
-		}
-
-		opts.BuildOptions.RepositoryDescription.Branches = append(opts.BuildOptions.RepositoryDescription.Branches, zoekt.RepositoryBranch{
-			Name:    b,
-			Version: commit.Hash.String(),
-		})
-
-		tree, err := commit.Tree()
-		if err != nil {
-			return err
-		}
-
-		files, subVersions, err := TreeToFiles(repo, tree, opts.BuildOptions.RepositoryDescription.URL, repoCache)
-		if err != nil {
-			return err
-		}
-		for k, v := range files {
-			repos[k] = v
-			branchMap[k] = append(branchMap[k], b)
-		}
-
-		branchVersions[b] = subVersions
-	}
-
-	if opts.Incremental && opts.BuildOptions.IncrementalSkipIndexing() {
-		return nil
-	}
-
-	reposByPath := map[string]BlobLocation{}
-	for key, location := range repos {
-		reposByPath[key.SubRepoPath] = location
-	}
-
-	opts.BuildOptions.SubRepositories = map[string]*zoekt.Repository{}
-	for path, location := range reposByPath {
-		tpl := opts.BuildOptions.RepositoryDescription
-		if path != "" {
-			tpl = zoekt.Repository{URL: location.URL.String()}
-			if err := SetTemplatesFromOrigin(&tpl, location.URL); err != nil {
-				log.Printf("setTemplatesFromOrigin(%s, %s): %s", path, location.URL, err)
-			}
-		}
-		opts.BuildOptions.SubRepositories[path] = &tpl
-	}
-	for _, br := range opts.BuildOptions.RepositoryDescription.Branches {
-		for path, repo := range opts.BuildOptions.SubRepositories {
-			id := branchVersions[br.Name][path]
-			repo.Branches = append(repo.Branches, zoekt.RepositoryBranch{
-				Name:    br.Name,
-				Version: id.String(),
-			})
-		}
-	}
-
-	builder, err := build.NewBuilder(opts.BuildOptions)
-	if err != nil {
-		return err
-	}
-	defer builder.Finish()
-
-	var names []string
-	fileKeys := map[string][]fileKey{}
-	for key := range repos {
-		n := key.FullPath()
-		fileKeys[n] = append(fileKeys[n], key)
-		names = append(names, n)
-	}
-
-	sort.Strings(names)
-	names = uniq(names)
-
-	for _, name := range names {
-		keys := fileKeys[name]
-
-		for _, key := range keys {
-			brs := branchMap[key]
-			blob, err := repos[key].Repo.BlobObject(key.ID)
-			if err != nil {
-				return err
-			}
-
-			if blob.Size > int64(opts.BuildOptions.SizeMax) && !opts.BuildOptions.IgnoreSizeMax(key.FullPath()) {
-				if err := builder.Add(zoekt.Document{
-					SkipReason:        fmt.Sprintf("file size %d exceeds maximum size %d", blob.Size, opts.BuildOptions.SizeMax),
-					Name:              key.FullPath(),
-					Branches:          brs,
-					SubRepositoryPath: key.SubRepoPath,
-				}); err != nil {
-					return err
-				}
-				continue
-			}
-
-			contents, err := blobContents(blob)
-			if err != nil {
-				return err
-			}
-			if err := builder.Add(zoekt.Document{
-				SubRepositoryPath: key.SubRepoPath,
-				Name:              key.FullPath(),
-				Content:           contents,
-				Branches:          brs,
-			}); err != nil {
-				return err
-			}
-		}
-	}
-	return builder.Finish()
-}
-
-func blobContents(blob *object.Blob) ([]byte, error) {
-	r, err := blob.Reader()
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-
-	var buf bytes.Buffer
-	buf.Grow(int(blob.Size))
-	_, err = buf.ReadFrom(r)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-func uniq(ss []string) []string {
-	result := ss[:0]
-	var last string
-	for i, s := range ss {
-		if i == 0 || s != last {
-			result = append(result, s)
-		}
-		last = s
-	}
-	return result
-}
diff --git a/gitindex/index_test.go b/gitindex/index_test.go
deleted file mode 100644
index bc24796..0000000
--- a/gitindex/index_test.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2021 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"testing"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-)
-
-func TestIndexEmptyRepo(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir %v", err)
-	}
-	defer os.RemoveAll(tmp)
-
-	cmd := exec.Command("git", "init", "-b", "master", "repo")
-	cmd.Dir = tmp
-
-	if err := cmd.Run(); err != nil {
-		t.Fatalf("cmd.Run: %v", err)
-	}
-
-	desc := zoekt.Repository{
-		Name: "repo",
-	}
-	opts := Options{
-		RepoDir: filepath.Join(tmp, "repo", ".git"),
-		BuildOptions: build.Options{
-			RepositoryDescription: desc,
-			IndexDir:              tmp,
-		},
-	}
-
-	if err := IndexGitRepo(opts); err != nil {
-		t.Fatalf("IndexGitRepo: %v", err)
-	}
-}
diff --git a/gitindex/repocache.go b/gitindex/repocache.go
deleted file mode 100644
index a8528f5..0000000
--- a/gitindex/repocache.go
+++ /dev/null
@@ -1,117 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-
-	git "github.com/go-git/go-git/v5"
-)
-
-// RepoCache is a set of repositories on the file system, named and
-// stored by URL.
-type RepoCache struct {
-	baseDir string
-
-	reposMu sync.Mutex
-	repos   map[string]*git.Repository
-}
-
-// NewRepoCache creates a new RepoCache rooted at the given directory.
-func NewRepoCache(dir string) *RepoCache {
-	return &RepoCache{
-		baseDir: dir,
-		repos:   make(map[string]*git.Repository),
-	}
-}
-
-func repoKeyStr(key string) string {
-	if !strings.HasSuffix(key, ".git") {
-		key += ".git"
-	}
-	return key
-}
-
-func repoKey(u *url.URL) string {
-	return repoKeyStr(filepath.Join(u.Host, u.Path))
-}
-
-// Path returns the absolute path of the bare repository.
-func Path(baseDir string, name string) string {
-	key := repoKeyStr(name)
-	return filepath.Join(baseDir, key)
-}
-
-func (rc *RepoCache) Path(u *url.URL) string {
-	key := repoKey(u)
-	return filepath.Join(rc.baseDir, key)
-}
-
-// Open opens a git repository. The cache retains a pointer to the
-// repository.
-func (rc *RepoCache) Open(u *url.URL) (*git.Repository, error) {
-	dir := rc.Path(u)
-	rc.reposMu.Lock()
-	defer rc.reposMu.Unlock()
-
-	key := repoKey(u)
-	r := rc.repos[key]
-	if r != nil {
-		return r, nil
-	}
-
-	repo, err := git.PlainOpen(dir)
-	if err == nil {
-		rc.repos[key] = repo
-	}
-	return repo, err
-}
-
-// ListRepos returns paths to repos on disk that start with the given
-// URL prefix. The paths are relative to baseDir, and typically
-// include a ".git" suffix.
-func ListRepos(baseDir string, u *url.URL) ([]string, error) {
-	key := filepath.Join(u.Host, u.Path)
-
-	var paths []string
-	walk := func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if !info.IsDir() {
-			return nil
-		}
-		if strings.HasSuffix(path, ".git") && !strings.HasSuffix(path, "/.git") {
-			_, err := git.PlainOpen(path)
-			if err == nil {
-				p, err := filepath.Rel(baseDir, path)
-				if err == nil {
-					paths = append(paths, p)
-				}
-			}
-			return filepath.SkipDir
-		}
-		return nil
-	}
-
-	if err := filepath.Walk(filepath.Join(baseDir, key), walk); err != nil {
-		return nil, err
-	}
-	return paths, nil
-}
diff --git a/gitindex/repocache_test.go b/gitindex/repocache_test.go
deleted file mode 100644
index 8d22d3a..0000000
--- a/gitindex/repocache_test.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"io/ioutil"
-	"net/url"
-	"os"
-	"reflect"
-	"sort"
-	"testing"
-)
-
-func TestListReposNonExistent(t *testing.T) {
-	u, err := url.Parse("https://gerrit.googlesource.com/")
-	if err != nil {
-		t.Fatalf("url.Parse: %v", err)
-	}
-
-	rs, err := ListRepos("/doesnotexist", u)
-	if err == nil {
-		t.Fatalf("ListRepos(/doesnotexist): %v", rs)
-	}
-}
-
-func TestListRepos(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir %v", err)
-	}
-	defer os.RemoveAll(tmp)
-	if err := createSubmoduleRepo(tmp); err != nil {
-		t.Fatalf("createSubmoduleRepo %v", err)
-	}
-
-	u, err := url.Parse("https://gerrit.googlesource.com/")
-	if err != nil {
-		t.Fatalf("url.Parse: %v", err)
-	}
-	rs, err := ListRepos(tmp, u)
-	if err != nil {
-		t.Fatalf("ListRepos(%s): %v", u, err)
-	}
-
-	want := []string{
-		"gerrit.googlesource.com/adir.git",
-		"gerrit.googlesource.com/bdir.git",
-		"gerrit.googlesource.com/sub/bdir.git",
-	}
-	sort.Strings(rs)
-
-	if !reflect.DeepEqual(rs, want) {
-		t.Fatalf("got %v, want %v", rs, want)
-	}
-}
diff --git a/gitindex/submodule.go b/gitindex/submodule.go
deleted file mode 100644
index 0836c89..0000000
--- a/gitindex/submodule.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"bytes"
-
-	"github.com/go-git/go-git/v5/plumbing/format/config"
-)
-
-// SubmoduleEntry represent one entry in a .gitmodules file
-type SubmoduleEntry struct {
-	Path   string
-	URL    string
-	Branch string
-}
-
-// ParseGitModules parses the contents of a .gitmodules file.
-func ParseGitModules(content []byte) (map[string]*SubmoduleEntry, error) {
-	dec := config.NewDecoder(bytes.NewBuffer(content))
-	cfg := &config.Config{}
-
-	if err := dec.Decode(cfg); err != nil {
-		return nil, err
-	}
-
-	result := map[string]*SubmoduleEntry{}
-	for _, s := range cfg.Sections {
-		if s.Name != "submodule" {
-			continue
-		}
-
-		for _, ss := range s.Subsections {
-			name := ss.Name
-			e := &SubmoduleEntry{}
-			for _, o := range ss.Options {
-				switch o.Key {
-				case "branch":
-					e.Branch = o.Value
-				case "path":
-					e.Path = o.Value
-				case "url":
-					e.URL = o.Value
-				}
-			}
-
-			result[name] = e
-		}
-	}
-
-	return result, nil
-}
diff --git a/gitindex/submodule_test.go b/gitindex/submodule_test.go
deleted file mode 100644
index ca7fef3..0000000
--- a/gitindex/submodule_test.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestParseGitModules(t *testing.T) {
-	testData := `[submodule "plugins/abc"]
-	path = plugins/abc
-	url = ../plugins/abc
-	branch = .`
-
-	got, err := ParseGitModules([]byte(testData))
-	if err != nil {
-		t.Fatalf("ParseGitModules: %T", err)
-	}
-
-	want := map[string]*SubmoduleEntry{
-		"plugins/abc": {
-			Path:   "plugins/abc",
-			URL:    "../plugins/abc",
-			Branch: ".",
-		},
-	}
-	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("got %v, want %v", got, want)
-	}
-}
diff --git a/gitindex/tree.go b/gitindex/tree.go
deleted file mode 100644
index 145cfd6..0000000
--- a/gitindex/tree.go
+++ /dev/null
@@ -1,217 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"fmt"
-	"io"
-	"log"
-	"net/url"
-	"path"
-	"path/filepath"
-	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/filemode"
-	"github.com/go-git/go-git/v5/plumbing/object"
-
-	git "github.com/go-git/go-git/v5"
-)
-
-// repoWalker walks a tree, recursing into submodules.
-type repoWalker struct {
-	repo *git.Repository
-
-	repoURL *url.URL
-	tree    map[fileKey]BlobLocation
-
-	// Path => SubmoduleEntry
-	submodules map[string]*SubmoduleEntry
-
-	// Path => commit SHA1
-	subRepoVersions map[string]plumbing.Hash
-	repoCache       *RepoCache
-}
-
-// subURL returns the URL for a submodule.
-func (w *repoWalker) subURL(relURL string) (*url.URL, error) {
-	if w.repoURL == nil {
-		return nil, fmt.Errorf("no URL for base repo")
-	}
-	if strings.HasPrefix(relURL, "../") {
-		u := *w.repoURL
-		u.Path = path.Join(u.Path, relURL)
-		return &u, nil
-	}
-
-	return url.Parse(relURL)
-}
-
-// newRepoWalker creates a new repoWalker.
-func newRepoWalker(r *git.Repository, repoURL string, repoCache *RepoCache) *repoWalker {
-	u, _ := url.Parse(repoURL)
-	return &repoWalker{
-		repo:            r,
-		repoURL:         u,
-		tree:            map[fileKey]BlobLocation{},
-		repoCache:       repoCache,
-		subRepoVersions: map[string]plumbing.Hash{},
-	}
-}
-
-// parseModuleMap initializes rw.submodules.
-func (rw *repoWalker) parseModuleMap(t *object.Tree) error {
-	modEntry, _ := t.File(".gitmodules")
-	if modEntry != nil {
-		c, err := blobContents(&modEntry.Blob)
-		if err != nil {
-			return err
-		}
-		mods, err := ParseGitModules(c)
-		if err != nil {
-			return err
-		}
-		rw.submodules = map[string]*SubmoduleEntry{}
-		for _, entry := range mods {
-			rw.submodules[entry.Path] = entry
-		}
-	}
-	return nil
-}
-
-// TreeToFiles fetches the blob SHA1s for a tree. If repoCache is
-// non-nil, recurse into submodules. In addition, it returns a mapping
-// that indicates in which repo each SHA1 can be found.
-func TreeToFiles(r *git.Repository, t *object.Tree,
-	repoURL string, repoCache *RepoCache) (map[fileKey]BlobLocation, map[string]plumbing.Hash, error) {
-	rw := newRepoWalker(r, repoURL, repoCache)
-
-	if err := rw.parseModuleMap(t); err != nil {
-		return nil, nil, err
-	}
-
-	tw := object.NewTreeWalker(t, true, make(map[plumbing.Hash]bool))
-	defer tw.Close()
-	for {
-		name, entry, err := tw.Next()
-		if err == io.EOF {
-			break
-		}
-		if err := rw.handleEntry(name, &entry); err != nil {
-			return nil, nil, err
-		}
-	}
-	return rw.tree, rw.subRepoVersions, nil
-}
-
-func (r *repoWalker) tryHandleSubmodule(p string, id *plumbing.Hash) error {
-	if err := r.handleSubmodule(p, id); err != nil {
-		log.Printf("submodule %s: ignoring error %v", p, err)
-	}
-	return nil
-}
-
-func (r *repoWalker) handleSubmodule(p string, id *plumbing.Hash) error {
-	submod := r.submodules[p]
-	if submod == nil {
-		return fmt.Errorf("no entry for submodule path %q", r.repoURL)
-	}
-
-	subURL, err := r.subURL(submod.URL)
-	if err != nil {
-		return err
-	}
-
-	subRepo, err := r.repoCache.Open(subURL)
-	if err != nil {
-		return err
-	}
-
-	obj, err := subRepo.CommitObject(*id)
-	if err != nil {
-		return err
-	}
-	tree, err := subRepo.TreeObject(obj.TreeHash)
-	if err != nil {
-		return err
-	}
-
-	r.subRepoVersions[p] = *id
-
-	subTree, subVersions, err := TreeToFiles(subRepo, tree, subURL.String(), r.repoCache)
-	if err != nil {
-		return err
-	}
-	for k, repo := range subTree {
-		r.tree[fileKey{
-			SubRepoPath: filepath.Join(p, k.SubRepoPath),
-			Path:        k.Path,
-			ID:          k.ID,
-		}] = repo
-	}
-	for k, v := range subVersions {
-		r.subRepoVersions[filepath.Join(p, k)] = v
-	}
-	return nil
-}
-
-func (r *repoWalker) handleEntry(p string, e *object.TreeEntry) error {
-	if e.Mode == filemode.Submodule && r.repoCache != nil {
-		if err := r.tryHandleSubmodule(p, &e.Hash); err != nil {
-			return fmt.Errorf("submodule %s: %v", p, err)
-		}
-	}
-
-	switch e.Mode {
-	case filemode.Regular, filemode.Executable:
-	default:
-		return nil
-	}
-
-	r.tree[fileKey{
-		Path: p,
-		ID:   e.Hash,
-	}] = BlobLocation{
-		Repo: r.repo,
-		URL:  r.repoURL,
-	}
-	return nil
-}
-
-// fileKey describes a blob at a location in the final tree. We also
-// record the subrepository from where it came.
-type fileKey struct {
-	SubRepoPath string
-	Path        string
-	ID          plumbing.Hash
-}
-
-func (k *fileKey) FullPath() string {
-	return filepath.Join(k.SubRepoPath, k.Path)
-}
-
-// BlobLocation holds data where a blob can be found.
-type BlobLocation struct {
-	Repo *git.Repository
-	URL  *url.URL
-}
-
-func (l *BlobLocation) Blob(id *plumbing.Hash) ([]byte, error) {
-	blob, err := l.Repo.BlobObject(*id)
-	if err != nil {
-		return nil, err
-	}
-	return blobContents(blob)
-}
diff --git a/gitindex/tree_test.go b/gitindex/tree_test.go
deleted file mode 100644
index eff31b3..0000000
--- a/gitindex/tree_test.go
+++ /dev/null
@@ -1,486 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gitindex
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io/ioutil"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"reflect"
-	"sort"
-	"testing"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/build"
-	"github.com/google/zoekt/query"
-	"github.com/google/zoekt/shards"
-)
-
-func createSubmoduleRepo(dir string) error {
-	if err := os.MkdirAll(dir, 0o755); err != nil {
-		return err
-	}
-	script := `mkdir adir bdir
-cd adir
-git init -b master
-mkdir subdir
-echo acont > afile
-echo sub-cont > subdir/sub-file
-git add afile subdir/sub-file
-git config user.email "you@example.com"
-git config user.name "Your Name"
-git commit -am amsg
-
-cd ..
-cd bdir
-git init -b master
-echo bcont > bfile
-git add bfile
-git config user.email "you@example.com"
-git config user.name "Your Name"
-git commit -am bmsg
-
-cd ../adir
-git submodule add --name bname -- ../bdir bname
-git commit -am bmodmsg
-cat .gitmodules
-cd ..
-mkdir gerrit.googlesource.com
-git clone --bare adir gerrit.googlesource.com/adir.git
-git clone --bare bdir gerrit.googlesource.com/bdir.git
-
-mkdir gerrit.googlesource.com/bogus.git
-mkdir gerrit.googlesource.com/sub
-git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
-
-cat << EOF  > gerrit.googlesource.com/adir.git/config
-[core]
-	repositoryformatversion = 0
-	filemode = true
-	bare = true
-[remote "origin"]
-	url = http://gerrit.googlesource.com/adir
-[branch "master"]
-	remote = origin
-	merge = refs/heads/master
-EOF
-`
-	cmd := exec.Command("/bin/sh", "-euxc", script)
-	cmd.Dir = dir
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("execution error: %v, output %s", err, out)
-	}
-	return nil
-}
-
-func TestFindGitRepos(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	if err := createSubmoduleRepo(dir); err != nil {
-		t.Error("createSubmoduleRepo", err)
-	}
-	repos, err := FindGitRepos(dir)
-	if err != nil {
-		t.Error("FindGitRepos", err)
-	}
-
-	got := map[string]bool{}
-	for _, r := range repos {
-		p, err := filepath.Rel(dir, r)
-		if err != nil {
-			t.Fatalf("Relative: %v", err)
-		}
-
-		got[p] = true
-	}
-
-	want := map[string]bool{
-		"gerrit.googlesource.com/bdir.git":     true,
-		"gerrit.googlesource.com/sub/bdir.git": true,
-		"adir/.git":                            true,
-		"bdir/.git":                            true,
-		"gerrit.googlesource.com/adir.git":     true,
-	}
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got %v want %v", got, want)
-	}
-}
-
-func TestTreeToFiles(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	if err := createSubmoduleRepo(dir); err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-
-	cache := NewRepoCache(dir)
-
-	aURL, _ := url.Parse("http://gerrit.googlesource.com/adir")
-	repo, err := cache.Open(aURL)
-	if err != nil {
-		t.Fatalf("Open: %v", err)
-	}
-
-	headRef, err := repo.Head()
-	if err != nil {
-		t.Fatalf("HEAD tree: %v", err)
-	}
-	commit, err := repo.CommitObject(headRef.Hash())
-	if err != nil {
-		t.Fatalf("commit obj HEAD: %v", err)
-	}
-
-	tree, err := repo.TreeObject(commit.TreeHash)
-	if err != nil {
-		t.Fatalf("AsTree: %v", err)
-	}
-
-	files, versions, err := TreeToFiles(repo, tree, aURL.String(), cache)
-	if err != nil {
-		t.Fatalf("TreeToFiles: %v", err)
-	}
-
-	bnameHash := versions["bname"]
-	if entry, err := tree.FindEntry("bname"); err != nil {
-		t.Fatalf("FindEntry %v", err)
-	} else if !bytes.Equal(bnameHash[:], entry.Hash[:]) {
-		t.Fatalf("got 'bname' versions %v, want %v", bnameHash, entry.Hash)
-	}
-
-	var paths []string
-	for k := range files {
-		paths = append(paths, k.FullPath())
-	}
-	sort.Strings(paths)
-
-	want := []string{".gitmodules", "afile", "bname/bfile", "subdir/sub-file"}
-	if !reflect.DeepEqual(paths, want) {
-		t.Errorf("got %v, want %v", paths, want)
-	}
-}
-
-func TestSubmoduleIndex(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	if err := createSubmoduleRepo(dir); err != nil {
-		t.Fatalf("createSubmoduleRepo: %v", err)
-	}
-
-	indexDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(indexDir)
-
-	buildOpts := build.Options{
-		IndexDir: indexDir,
-	}
-	opts := Options{
-		RepoDir:      filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
-		BuildOptions: buildOpts,
-		BranchPrefix: "refs/heads/",
-		Branches:     []string{"master"},
-		Submodules:   true,
-		Incremental:  true,
-		RepoCacheDir: dir,
-	}
-	if err := IndexGitRepo(opts); err != nil {
-		t.Fatalf("IndexGitRepo: %v", err)
-	}
-
-	searcher, err := shards.NewDirectorySearcher(indexDir)
-	if err != nil {
-		t.Fatal("NewDirectorySearcher", err)
-	}
-	defer searcher.Close()
-
-	results, err := searcher.Search(context.Background(),
-		&query.Substring{Pattern: "bcont"},
-		&zoekt.SearchOptions{})
-	if err != nil {
-		t.Fatal("Search", err)
-	}
-
-	if len(results.Files) != 1 {
-		t.Fatalf("got search result %v, want 1 file", results.Files)
-	}
-
-	file := results.Files[0]
-	if got, want := file.SubRepositoryName, "gerrit.googlesource.com/bdir"; got != want {
-		t.Errorf("got subrepo name %q, want %q", got, want)
-	}
-	if got, want := file.SubRepositoryPath, "bname"; got != want {
-		t.Errorf("got subrepo path %q, want %q", got, want)
-	}
-
-	subVersion := file.Version
-	if len(subVersion) != 40 {
-		t.Fatalf("got %q, want hex sha1", subVersion)
-	}
-
-	if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
-		t.Fatalf("Search('acont'): %v", err)
-	} else if len(results.Files) != 1 {
-		t.Errorf("got %v, want 1 result", results.Files)
-	} else if f := results.Files[0]; f.Version == subVersion {
-		t.Errorf("version in super repo matched version is subrepo.")
-	}
-}
-
-func TestAllowMissingBranch(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-	if err := createSubmoduleRepo(dir); err != nil {
-		t.Fatalf("createSubmoduleRepo: %v", err)
-	}
-
-	indexDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(indexDir)
-
-	buildOpts := build.Options{
-		IndexDir: indexDir,
-	}
-
-	opts := Options{
-		RepoDir:      filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
-		BuildOptions: buildOpts,
-		BranchPrefix: "refs/heads/",
-		Branches:     []string{"master", "nonexist"},
-		Submodules:   true,
-		Incremental:  true,
-		RepoCacheDir: dir,
-	}
-	if err := IndexGitRepo(opts); err == nil {
-		t.Fatalf("IndexGitRepo(nonexist) succeeded")
-	}
-	opts.AllowMissingBranch = true
-	if err := IndexGitRepo(opts); err != nil {
-		t.Fatalf("IndexGitRepo(nonexist, allow): %v", err)
-	}
-}
-
-func createMultibranchRepo(dir string) error {
-	if err := os.MkdirAll(dir, 0o755); err != nil {
-		return err
-	}
-	script := `mkdir repo
-cd repo
-git init -b master
-mkdir subdir
-echo acont > afile
-echo sub-cont > subdir/sub-file
-git add afile subdir/sub-file
-git config user.email "you@example.com"
-git config user.name "Your Name"
-git commit -am amsg
-
-git branch branchdir/a
-
-echo acont >> afile
-git add afile subdir/sub-file
-git commit -am amsg
-
-git branch branchdir/b
-
-git branch c
-
-git update-ref refs/meta/config HEAD
-`
-	cmd := exec.Command("/bin/sh", "-euxc", script)
-	cmd.Dir = dir
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("execution error: %v, output %s", err, out)
-	}
-	return nil
-}
-
-func TestBranchWildcard(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	if err := createMultibranchRepo(dir); err != nil {
-		t.Fatalf("createMultibranchRepo: %v", err)
-	}
-
-	indexDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(indexDir)
-
-	buildOpts := build.Options{
-		IndexDir: indexDir,
-		RepositoryDescription: zoekt.Repository{
-			Name: "repo",
-		},
-	}
-	buildOpts.SetDefaults()
-
-	opts := Options{
-		RepoDir:      filepath.Join(dir + "/repo"),
-		BuildOptions: buildOpts,
-		BranchPrefix: "refs/heads",
-		Branches:     []string{"branchdir/*"},
-		Submodules:   true,
-		Incremental:  true,
-	}
-	if err := IndexGitRepo(opts); err != nil {
-		t.Fatalf("IndexGitRepo: %v", err)
-	}
-
-	searcher, err := shards.NewDirectorySearcher(indexDir)
-	if err != nil {
-		t.Fatal("NewDirectorySearcher", err)
-	}
-	defer searcher.Close()
-
-	if rlist, err := searcher.List(context.Background(), &query.Repo{Pattern: ""}); err != nil {
-		t.Fatalf("List(): %v", err)
-	} else if len(rlist.Repos) != 1 {
-		t.Errorf("got %v, want 1 result", rlist.Repos)
-	} else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 2 {
-		t.Errorf("got branches %v, want 2", repo.Repository.Branches)
-	} else if repo := rlist.Repos[0]; repo.Stats.Documents != 3 {
-		t.Errorf("got document count %d, want 3", repo.Stats.Documents)
-	}
-}
-
-func TestSkipSubmodules(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	if err := createSubmoduleRepo(dir); err != nil {
-		t.Fatalf("createMultibranchRepo: %v", err)
-	}
-
-	indexDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(indexDir)
-
-	buildOpts := build.Options{
-		IndexDir: indexDir,
-		RepositoryDescription: zoekt.Repository{
-			Name: "gerrit.googlesource.com/adir",
-		},
-	}
-	if err := os.Rename(dir+"/gerrit.googlesource.com/bdir.git",
-		dir+"/gerrit.googlesource.com/notexist.git"); err != nil {
-		t.Fatalf("Rename: %v", err)
-	}
-
-	opts := Options{
-		RepoDir:      filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
-		BuildOptions: buildOpts,
-		BranchPrefix: "refs/heads",
-		Branches:     []string{"master"},
-		Submodules:   false,
-	}
-	if err := IndexGitRepo(opts); err != nil {
-		t.Fatalf("IndexGitRepo: %v", err)
-	}
-}
-
-func TestFullAndShortRefNames(t *testing.T) {
-	dir, err := ioutil.TempDir("", "git")
-	if err != nil {
-		t.Fatalf("TempDir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	if err := createMultibranchRepo(dir); err != nil {
-		t.Fatalf("createMultibranchRepo: %v", err)
-	}
-
-	indexDir, err := ioutil.TempDir("", "index-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	os.RemoveAll(indexDir)
-
-	buildOpts := build.Options{
-		IndexDir: indexDir,
-		RepositoryDescription: zoekt.Repository{
-			Name: "repo",
-		},
-	}
-	buildOpts.SetDefaults()
-
-	opts := Options{
-		RepoDir:            filepath.Join(dir + "/repo"),
-		BuildOptions:       buildOpts,
-		BranchPrefix:       "refs/heads",
-		Branches:           []string{"refs/heads/master", "branchdir/a", "refs/meta/config"},
-		Submodules:         false,
-		Incremental:        false,
-		AllowMissingBranch: false,
-	}
-	if err := IndexGitRepo(opts); err != nil {
-		t.Fatalf("IndexGitRepo: %v", err)
-	}
-
-	searcher, err := shards.NewDirectorySearcher(indexDir)
-	if err != nil {
-		t.Fatal("NewDirectorySearcher", err)
-	}
-	defer searcher.Close()
-
-	if rlist, err := searcher.List(context.Background(), &query.Repo{Pattern: ""}); err != nil {
-		t.Fatalf("List(): %v", err)
-	} else if len(rlist.Repos) != 1 {
-		t.Errorf("got %v, want 1 result", rlist.Repos)
-	} else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 3 {
-		t.Errorf("got branches %v, want 3", repo.Repository.Branches)
-	}
-}
-
-func TestUniq(t *testing.T) {
-	in := []string{"a", "b", "b", "c", "c"}
-	want := []string{"a", "b", "c"}
-	got := uniq(in)
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
diff --git a/go.mod b/go.mod
deleted file mode 100644
index 90ae146..0000000
--- a/go.mod
+++ /dev/null
@@ -1,27 +0,0 @@
-module github.com/google/zoekt
-
-go 1.13
-
-require (
-	github.com/andygrunwald/go-gerrit v0.0.0-20191101112536-3f5e365ccf57
-	github.com/bmatcuk/doublestar v1.3.4
-	github.com/fsnotify/fsnotify v1.4.7
-	github.com/gfleury/go-bitbucket-v1 v0.0.0-20200104105711-ddbafbb02522
-	github.com/go-git/go-git/v5 v5.0.0
-	github.com/golang/protobuf v1.3.3 // indirect
-	github.com/google/go-cmp v0.5.5
-	github.com/google/go-github/v27 v27.0.6
-	github.com/google/slothfs v0.0.0-20190417171004-6b42407d9230
-	github.com/kylelemons/godebug v1.1.0
-	github.com/mitchellh/mapstructure v1.1.2 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.5.1
-	github.com/xanzy/go-gitlab v0.25.0
-	go.uber.org/automaxprocs v1.3.0
-	golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	google.golang.org/appengine v1.6.5 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	humungus.tedunangst.com/r/gerc v0.1.2
-)
diff --git a/go.sum b/go.sum
deleted file mode 100644
index 4b1a482..0000000
--- a/go.sum
+++ /dev/null
@@ -1,212 +0,0 @@
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
-github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/andygrunwald/go-gerrit v0.0.0-20191101112536-3f5e365ccf57 h1:wtSQ14h8qAUezER6QPfYmCh5+W5Ly1lVruhm/QeOVUE=
-github.com/andygrunwald/go-gerrit v0.0.0-20191101112536-3f5e365ccf57/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
-github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
-github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
-github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gfleury/go-bitbucket-v1 v0.0.0-20200104105711-ddbafbb02522 h1:hrfa10FjSX7jqupn/o8vyEsJ807SyKvuf+iFiEytTN8=
-github.com/gfleury/go-bitbucket-v1 v0.0.0-20200104105711-ddbafbb02522/go.mod h1:Se0U4YUmRkRAOh8kD7KXz+3VCUBmvTFcdWP2QYYRjjc=
-github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
-github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
-github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
-github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
-github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
-github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-git-fixtures/v4 v4.0.1 h1:q+IFMfLx200Q3scvt2hN79JsEzy4AmBTp/pqnefH+Bc=
-github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
-github.com/go-git/go-git/v5 v5.0.0 h1:k5RWPm4iJwYtfWoxIJy4wJX9ON7ihPeZZYC1fLYDnpg=
-github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmClfZwtUVA=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-github/v27 v27.0.6 h1:oiOZuBmGHvrGM1X9uNUAUlLgp5r1UUO/M/KnbHnLRlQ=
-github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/slothfs v0.0.0-20190417171004-6b42407d9230 h1:iBLrJ79cF90CZmpskySqhPvzrWr9njBYEsOZubXLZlc=
-github.com/google/slothfs v0.0.0-20190417171004-6b42407d9230/go.mod h1:kzvK/MFjZSNdFgc1tCZML3E1nVvnB4/npSKEuvMoECU=
-github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
-github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
-github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA=
-github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
-github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
-github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/xanzy/go-gitlab v0.25.0 h1:G5aTZeqZd66Q6qMVieBfmHBsPpF0jY92zCLAMpULe3I=
-github.com/xanzy/go-gitlab v0.25.0/go.mod h1:t4Bmvnxj7k37S4Y17lfLx+nLqkf/oQwT2HagfWKv5Og=
-github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
-github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
-go.uber.org/automaxprocs v1.3.0 h1:II28aZoGdaglS5vVNnspf28lnZpXScxtIozx1lAjdb0=
-go.uber.org/automaxprocs v1.3.0/go.mod h1:9CWT6lKIep8U41DDaPiH6eFscnTyjfTANNQNx6LrIcA=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6 h1:0PC75Fz/kyMGhL0e1QnypqK2kQMqKt9csD1GnMJR+Zk=
-golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
-gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-humungus.tedunangst.com/r/gerc v0.1.2 h1:eW5yTbRLFFWSu/RpTdkLxaVPlNlFUzxc02VQsftzg64=
-humungus.tedunangst.com/r/gerc v0.1.2/go.mod h1:tuYnDVV3WEGI9NEX5/3Iz5xVNimFzN4+83qZvFf/GUg=
diff --git a/hititer.go b/hititer.go
deleted file mode 100644
index 356c505..0000000
--- a/hititer.go
+++ /dev/null
@@ -1,259 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"encoding/binary"
-	"fmt"
-)
-
-// hitIterator finds potential search matches, measured in offsets of
-// the concatenation of all documents.
-type hitIterator interface {
-	// Return the first hit, or maxUInt32 if none.
-	first() uint32
-
-	// Skip until past limit. The argument maxUInt32 should be
-	// treated specially.
-	next(limit uint32)
-
-	// Return how many bytes were read.
-	updateStats(s *Stats)
-}
-
-// distanceHitIterator looks for hits at a fixed distance apart.
-type distanceHitIterator struct {
-	started  bool
-	distance uint32
-	i1       hitIterator
-	i2       hitIterator
-}
-
-func (i *distanceHitIterator) String() string {
-	return fmt.Sprintf("dist(%d, %v, %v)", i.distance, i.i1, i.i2)
-}
-
-func (i *distanceHitIterator) findNext() {
-	for {
-		var p1, p2 uint32
-		p1 = i.i1.first()
-		p2 = i.i2.first()
-		if p1 == maxUInt32 || p2 == maxUInt32 {
-			i.i1.next(maxUInt32)
-			break
-		}
-
-		if p1+i.distance < p2 {
-			i.i1.next(p2 - i.distance - 1)
-		} else if p1+i.distance > p2 {
-			i.i2.next(p1 + i.distance - 1)
-		} else {
-			break
-		}
-	}
-}
-
-func (i *distanceHitIterator) first() uint32 {
-	if !i.started {
-		i.findNext()
-		i.started = true
-	}
-	return i.i1.first()
-}
-
-func (i *distanceHitIterator) updateStats(s *Stats) {
-	i.i1.updateStats(s)
-	i.i2.updateStats(s)
-}
-
-func (i *distanceHitIterator) next(limit uint32) {
-	i.i1.next(limit)
-	l2 := limit + i.distance
-
-	if l2 < limit { // overflow.
-		l2 = maxUInt32
-	}
-	i.i2.next(l2)
-	i.findNext()
-}
-
-func (d *indexData) newDistanceTrigramIter(ng1, ng2 ngram, dist uint32, caseSensitive, fileName bool) (hitIterator, error) {
-	if dist == 0 {
-		return nil, fmt.Errorf("d == 0")
-	}
-
-	i1, err := d.trigramHitIterator(ng1, caseSensitive, fileName)
-	if err != nil {
-		return nil, err
-	}
-	i2, err := d.trigramHitIterator(ng2, caseSensitive, fileName)
-	if err != nil {
-		return nil, err
-	}
-	return &distanceHitIterator{
-		i1:       i1,
-		i2:       i2,
-		distance: dist,
-	}, nil
-}
-
-func (d *indexData) trigramHitIterator(ng ngram, caseSensitive, fileName bool) (hitIterator, error) {
-	variants := []ngram{ng}
-	if !caseSensitive {
-		variants = generateCaseNgrams(ng)
-	}
-
-	iters := make([]hitIterator, 0, len(variants))
-	for _, v := range variants {
-		if fileName {
-			blob := d.fileNameNgrams[v]
-			if len(blob) > 0 {
-				iters = append(iters, &inMemoryIterator{
-					d.fileNameNgrams[v],
-					v,
-				})
-			}
-			continue
-		}
-
-		sec := d.ngrams[v]
-		blob, err := d.readSectionBlob(sec)
-		if err != nil {
-			return nil, err
-		}
-		if len(blob) > 0 {
-			iters = append(iters, newCompressedPostingIterator(blob, v))
-		}
-	}
-
-	if len(iters) == 1 {
-		return iters[0], nil
-	}
-	return &mergingIterator{
-		iters: iters,
-	}, nil
-}
-
-// inMemoryIterator is hitIterator that goes over an in-memory uint32 posting list.
-type inMemoryIterator struct {
-	postings []uint32
-	what     ngram
-}
-
-func (i *inMemoryIterator) String() string {
-	return fmt.Sprintf("mem(%s):%v", i.what, i.postings)
-}
-
-func (i *inMemoryIterator) first() uint32 {
-	if len(i.postings) > 0 {
-		return i.postings[0]
-	}
-	return maxUInt32
-}
-
-func (i *inMemoryIterator) updateStats(s *Stats) {
-}
-
-func (i *inMemoryIterator) next(limit uint32) {
-	if limit == maxUInt32 {
-		i.postings = nil
-	}
-
-	for len(i.postings) > 0 && i.postings[0] <= limit {
-		i.postings = i.postings[1:]
-	}
-}
-
-// compressedPostingIterator goes over a delta varint encoded posting
-// list.
-type compressedPostingIterator struct {
-	blob, orig []byte
-	_first     uint32
-	what       ngram
-}
-
-func newCompressedPostingIterator(b []byte, w ngram) *compressedPostingIterator {
-	d, sz := binary.Uvarint(b)
-	return &compressedPostingIterator{
-		_first: uint32(d),
-		blob:   b[sz:],
-		orig:   b,
-		what:   w,
-	}
-}
-
-func (i *compressedPostingIterator) String() string {
-	return fmt.Sprintf("compressed(%s, %d, [%d bytes])", i.what, i._first, len(i.blob))
-}
-
-func (i *compressedPostingIterator) first() uint32 {
-	return i._first
-}
-
-func (i *compressedPostingIterator) next(limit uint32) {
-	if limit == maxUInt32 {
-		i.blob = nil
-		i._first = maxUInt32
-		return
-	}
-
-	for i._first <= limit && len(i.blob) > 0 {
-		delta, sz := binary.Uvarint(i.blob)
-		i._first += uint32(delta)
-		i.blob = i.blob[sz:]
-	}
-
-	if i._first <= limit && len(i.blob) == 0 {
-		i._first = maxUInt32
-	}
-}
-
-func (i *compressedPostingIterator) updateStats(s *Stats) {
-	s.IndexBytesLoaded += int64(len(i.orig) - len(i.blob))
-}
-
-// mergingIterator forms the merge of a set of hitIterators, to
-// implement an OR operation at the hit level.
-type mergingIterator struct {
-	iters []hitIterator
-}
-
-func (i *mergingIterator) String() string {
-	return fmt.Sprintf("merge:%v", i.iters)
-}
-
-func (i *mergingIterator) updateStats(s *Stats) {
-	for _, j := range i.iters {
-		j.updateStats(s)
-	}
-}
-
-func (i *mergingIterator) first() uint32 {
-	r := uint32(maxUInt32)
-	for _, j := range i.iters {
-		f := j.first()
-		if f < r {
-			r = f
-		}
-	}
-
-	return r
-}
-
-func (i *mergingIterator) next(limit uint32) {
-	for _, j := range i.iters {
-		j.next(limit)
-	}
-}
diff --git a/hititer_test.go b/hititer_test.go
deleted file mode 100644
index 0c276c1..0000000
--- a/hititer_test.go
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright 2019 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"fmt"
-	"math/rand"
-	"reflect"
-	"sort"
-	"testing"
-	"testing/quick"
-
-	"github.com/google/go-cmp/cmp"
-)
-
-func TestCompressedPostingIterator_limit(t *testing.T) {
-	f := func(nums, limits []uint32) bool {
-		if len(nums) == 0 || len(limits) == 0 {
-			return true
-		}
-
-		nums = sortedUnique(nums)
-		sort.Slice(limits, func(i, j int) bool { return limits[i] < limits[j] })
-
-		want := doHitIterator(&inMemoryIterator{postings: nums}, limits)
-
-		it := newCompressedPostingIterator(toDeltas(nums), stringToNGram("abc"))
-		got := doHitIterator(it, limits)
-		if !reflect.DeepEqual(want, got) {
-			t.Log(cmp.Diff(want, got))
-			return false
-		}
-		return true
-	}
-	if err := quick.Check(f, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func doHitIterator(it hitIterator, limits []uint32) []uint32 {
-	var nums []uint32
-	for _, limit := range limits {
-		it.next(limit)
-		nums = append(nums, it.first())
-	}
-	return nums
-}
-
-func BenchmarkCompressedPostingIterator(b *testing.B) {
-	cases := []struct{ size, limitSize int }{
-		{100, 50},
-		{10000, 100},
-		{10000, 1000},
-		{10000, 10000},
-		{100000, 100},
-		{100000, 1000},
-		{100000, 10000},
-		{100000, 100000},
-	}
-	for _, tt := range cases {
-		b.Run(fmt.Sprintf("%d_%d", tt.size, tt.limitSize), func(b *testing.B) {
-			benchmarkCompressedPostingIterator(b, tt.size, tt.limitSize)
-		})
-	}
-}
-
-func benchmarkCompressedPostingIterator(b *testing.B, size, limitsSize int) {
-	nums := genUints32(size)
-	limits := genUints32(limitsSize)
-
-	nums = sortedUnique(nums)
-	sort.Slice(limits, func(i, j int) bool { return limits[i] < limits[j] })
-
-	ng := stringToNGram("abc")
-	deltas := toDeltas(nums)
-
-	b.ResetTimer()
-
-	for n := 0; n < b.N; n++ {
-		it := newCompressedPostingIterator(deltas, ng)
-		for _, limit := range limits {
-			it.next(limit)
-			_ = it.first()
-		}
-		var s Stats
-		it.updateStats(&s)
-		b.SetBytes(s.IndexBytesLoaded)
-	}
-}
-
-func genUints32(size int) []uint32 {
-	// Deterministic for benchmarks
-	r := rand.New(rand.NewSource(int64(size)))
-	nums := make([]uint32, size)
-	for i := range nums {
-		nums[i] = r.Uint32()
-	}
-	return nums
-}
diff --git a/index_test.go b/index_test.go
deleted file mode 100644
index 0f84197..0000000
--- a/index_test.go
+++ /dev/null
@@ -1,1938 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"reflect"
-	"regexp/syntax"
-	"strings"
-	"testing"
-
-	"github.com/kylelemons/godebug/pretty"
-
-	"github.com/google/zoekt/query"
-)
-
-func clearScores(r *SearchResult) {
-	for i := range r.Files {
-		r.Files[i].Score = 0.0
-		for j := range r.Files[i].LineMatches {
-			r.Files[i].LineMatches[j].Score = 0.0
-		}
-		r.Files[i].Checksum = nil
-		r.Files[i].Debug = ""
-	}
-}
-
-func testIndexBuilder(t *testing.T, repo *Repository, docs ...Document) *IndexBuilder {
-	b, err := NewIndexBuilder(repo)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	for i, d := range docs {
-		if err := b.Add(d); err != nil {
-			t.Fatalf("Add %d: %v", i, err)
-		}
-	}
-	return b
-}
-
-func TestBoundary(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: []byte("x the")},
-		Document{Name: "f1", Content: []byte("reader")})
-	res := searchForTest(t, b, &query.Substring{Pattern: "there"})
-	if len(res.Files) > 0 {
-		t.Fatalf("got %v, want no matches", res.Files)
-	}
-}
-
-func TestDocSectionInvalid(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-	doc := Document{
-		Name:    "f1",
-		Content: []byte("01234567890123"),
-		Symbols: []DocumentSection{{5, 8}, {7, 9}},
-	}
-
-	if err := b.Add(doc); err == nil {
-		t.Errorf("overlapping doc sections should fail")
-	}
-
-	doc = Document{
-		Name:    "f1",
-		Content: []byte("01234567890123"),
-		Symbols: []DocumentSection{{0, 20}},
-	}
-
-	if err := b.Add(doc); err == nil {
-		t.Errorf("doc sections beyond EOF should fail")
-	}
-}
-
-func TestBasic(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f2",
-			Content: []byte("to carry water in the no later bla"),
-			// ------------- 0123456789012345678901234567890123456789
-		})
-
-	res := searchForTest(t, b, &query.Substring{
-		Pattern:       "water",
-		CaseSensitive: true,
-	})
-	fmatches := res.Files
-	if len(fmatches) != 1 || len(fmatches[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 matches", fmatches)
-	}
-
-	got := fmt.Sprintf("%s:%d", fmatches[0].FileName, fmatches[0].LineMatches[0].LineFragments[0].Offset)
-	want := "f2:9"
-	if got != want {
-		t.Errorf("1: got %s, want %s", got, want)
-	}
-}
-
-func TestEmptyIndex(t *testing.T) {
-	b := testIndexBuilder(t, nil)
-	searcher := searcherForTest(t, b)
-
-	var opts SearchOptions
-	if _, err := searcher.Search(context.Background(), &query.Substring{}, &opts); err != nil {
-		t.Fatalf("Search: %v", err)
-	}
-
-	if _, err := searcher.List(context.Background(), &query.Repo{}); err != nil {
-		t.Fatalf("List: %v", err)
-	}
-
-	if _, err := searcher.Search(context.Background(), &query.Substring{Pattern: "java", FileName: true}, &opts); err != nil {
-		t.Fatalf("Search: %v", err)
-	}
-}
-
-type memSeeker struct {
-	data []byte
-}
-
-func (s *memSeeker) Name() string {
-	return "memseeker"
-}
-
-func (s *memSeeker) Close() {}
-func (s *memSeeker) Read(off, sz uint32) ([]byte, error) {
-	return s.data[off : off+sz], nil
-}
-
-func (s *memSeeker) Size() (uint32, error) {
-	return uint32(len(s.data)), nil
-}
-
-func TestNewlines(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{Name: "filename", Content: []byte("line1\nline2\nbla")})
-
-	sres := searchForTest(t, b, &query.Substring{Pattern: "ne2"})
-
-	matches := sres.Files
-	want := []FileMatch{{
-		FileName: "filename",
-		LineMatches: []LineMatch{
-			{
-				LineFragments: []LineFragmentMatch{{
-					Offset:      8,
-					LineOffset:  2,
-					MatchLength: 3,
-				}},
-				Line:       []byte("line2"),
-				LineStart:  6,
-				LineEnd:    11,
-				LineNumber: 2,
-			},
-		},
-	}}
-
-	if !reflect.DeepEqual(matches, want) {
-		t.Errorf("got %v, want %v", matches, want)
-	}
-}
-
-// A result spanning multiple lines should have LineMatches that only cover
-// single lines.
-func TestQueryNewlines(t *testing.T) {
-	text := "line1\nline2\nbla"
-	b := testIndexBuilder(t, nil,
-		Document{Name: "filename", Content: []byte(text)})
-	sres := searchForTest(t, b, &query.Substring{Pattern: "ine2\nbla"})
-	matches := sres.Files
-	if len(matches) != 1 {
-		t.Fatalf("got %d file matches, want exactly one", len(matches))
-	}
-	m := matches[0]
-	if len(m.LineMatches) != 2 {
-		t.Fatalf("got %d line matches, want exactly two", len(m.LineMatches))
-	}
-}
-
-func searchForTest(t *testing.T, b *IndexBuilder, q query.Q, o ...SearchOptions) *SearchResult {
-	searcher := searcherForTest(t, b)
-	var opts SearchOptions
-	if len(o) > 0 {
-		opts = o[0]
-	}
-	res, err := searcher.Search(context.Background(), q, &opts)
-	if err != nil {
-		t.Fatalf("Search(%s): %v", q, err)
-	}
-	clearScores(res)
-	return res
-}
-
-func searcherForTest(t *testing.T, b *IndexBuilder) Searcher {
-	var buf bytes.Buffer
-	b.Write(&buf)
-	f := &memSeeker{buf.Bytes()}
-
-	searcher, err := NewSearcher(f)
-	if err != nil {
-		t.Fatalf("NewSearcher: %v", err)
-	}
-
-	return searcher
-}
-
-func TestFileBasedSearch(t *testing.T) {
-	c1 := []byte("I love bananas without skin")
-	// -----------0123456789012345678901234567890123456789
-	c2 := []byte("In Dutch, ananas means pineapple")
-	// -----------0123456789012345678901234567890123456789
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: c1},
-		Document{Name: "f2", Content: c2},
-	)
-	sres := searchForTest(t, b, &query.Substring{
-		CaseSensitive: false,
-		Pattern:       "ananas",
-	})
-
-	matches := sres.Files
-	if len(matches) != 2 {
-		t.Fatalf("got %v, want 2 matches", matches)
-	}
-	if matches[0].FileName != "f2" || matches[1].FileName != "f1" {
-		t.Fatalf("got %v, want matches {f1,f2}", matches)
-	}
-	if matches[0].LineMatches[0].LineFragments[0].Offset != 10 || matches[1].LineMatches[0].LineFragments[0].Offset != 8 {
-		t.Fatalf("got %#v, want offsets 10,8", matches)
-	}
-}
-
-func TestCaseFold(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: []byte("I love BaNaNAS.")},
-		// ---------- 012345678901234567890123456
-	)
-	sres := searchForTest(t, b, &query.Substring{
-		Pattern:       "bananas",
-		CaseSensitive: true,
-	})
-	matches := sres.Files
-	if len(matches) != 0 {
-		t.Errorf("foldcase: got %#v, want 0 matches", matches)
-	}
-
-	sres = searchForTest(t, b,
-		&query.Substring{
-			Pattern:       "BaNaNAS",
-			CaseSensitive: true,
-		})
-	matches = sres.Files
-	if len(matches) != 1 {
-		t.Errorf("no foldcase: got %v, want 1 matches", matches)
-	} else if matches[0].LineMatches[0].LineFragments[0].Offset != 7 {
-		t.Errorf("foldcase: got %v, want offsets 7", matches)
-	}
-}
-
-func TestAndSearch(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("f1", []byte("x banana y"))
-	b.AddFile("f2", []byte("x apple y"))
-	b.AddFile("f3", []byte("x banana apple y"))
-	// ---------------------0123456789012345
-	sres := searchForTest(t, b, query.NewAnd(
-		&query.Substring{
-			Pattern: "banana",
-		},
-		&query.Substring{
-			Pattern: "apple",
-		},
-	))
-	matches := sres.Files
-	if len(matches) != 1 || len(matches[0].LineMatches) != 1 || len(matches[0].LineMatches[0].LineFragments) != 2 {
-		t.Fatalf("got %#v, want 1 match with 2 fragments", matches)
-	}
-
-	if matches[0].LineMatches[0].LineFragments[0].Offset != 2 || matches[0].LineMatches[0].LineFragments[1].Offset != 9 {
-		t.Fatalf("got %#v, want offsets 2,9", matches)
-	}
-
-	wantStats := Stats{
-		FilesLoaded:        1,
-		ContentBytesLoaded: 18,
-		IndexBytesLoaded:   8,
-		NgramMatches:       3, // we look at doc 1, because it's max(0,1) due to AND
-		MatchCount:         1,
-		FileCount:          1,
-		FilesConsidered:    2,
-	}
-	if diff := pretty.Compare(wantStats, sres.Stats); diff != "" {
-		t.Errorf("got stats diff %s", diff)
-	}
-}
-
-func TestAndNegateSearch(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("f1", []byte("x banana y"))
-	b.AddFile("f4", []byte("x banana apple y"))
-	// ---------------------0123456789012345
-	sres := searchForTest(t, b, query.NewAnd(
-		&query.Substring{
-			Pattern: "banana",
-		},
-		&query.Not{Child: &query.Substring{
-			Pattern: "apple",
-		}}))
-
-	matches := sres.Files
-
-	if len(matches) != 1 || len(matches[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 match", matches)
-	}
-	if matches[0].FileName != "f1" {
-		t.Fatalf("got match %#v, want FileName: f1", matches[0])
-	}
-	if matches[0].LineMatches[0].LineFragments[0].Offset != 2 {
-		t.Fatalf("got %v, want offsets 2,9", matches)
-	}
-}
-
-func TestNegativeMatchesOnlyShortcut(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("f1", []byte("x banana y"))
-	b.AddFile("f2", []byte("x appelmoes y"))
-	b.AddFile("f3", []byte("x appelmoes y"))
-	b.AddFile("f3", []byte("x appelmoes y"))
-
-	sres := searchForTest(t, b, query.NewAnd(
-		&query.Substring{
-			Pattern: "banana",
-		},
-		&query.Not{Child: &query.Substring{
-			Pattern: "appel",
-		}}))
-
-	if sres.Stats.FilesConsidered != 1 {
-		t.Errorf("got %#v, want FilesConsidered: 1", sres.Stats)
-	}
-}
-
-func TestFileSearch(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("banzana", []byte("x orange y"))
-	// --------0123456
-	b.AddFile("banana", []byte("x apple y"))
-	// --------789012
-	sres := searchForTest(t, b, &query.Substring{
-		Pattern:  "anan",
-		FileName: true,
-	})
-
-	matches := sres.Files
-	if len(matches) != 1 || len(matches[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 match", matches)
-	}
-
-	got := matches[0].LineMatches[0]
-	want := LineMatch{
-		Line: []byte("banana"),
-		LineFragments: []LineFragmentMatch{{
-			Offset:      1,
-			LineOffset:  1,
-			MatchLength: 4,
-		}},
-		FileName: true,
-	}
-
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got %#v, want %#v", got, want)
-	}
-}
-
-func TestFileCase(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("BANANA", []byte("x orange y"))
-	sres := searchForTest(t, b, &query.Substring{
-		Pattern:  "banana",
-		FileName: true,
-	})
-
-	matches := sres.Files
-	if len(matches) != 1 || matches[0].FileName != "BANANA" {
-		t.Fatalf("got %v, want 1 match 'BANANA'", matches)
-	}
-}
-
-func TestFileRegexpSearchBruteForce(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("banzana", []byte("x orange y"))
-	// --------------------------0123456879
-	b.AddFile("banana", []byte("x apple y"))
-	sres := searchForTest(t, b, &query.Regexp{
-		Regexp:   mustParseRE("[qn][zx]"),
-		FileName: true,
-	})
-
-	matches := sres.Files
-	if len(matches) != 1 || matches[0].FileName != "banzana" {
-		t.Fatalf("got %v, want 1 match on 'banzana'", matches)
-	}
-}
-
-func TestFileRegexpSearchShortString(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("banana.py", []byte("x orange y"))
-	sres := searchForTest(t, b, &query.Regexp{
-		Regexp:   mustParseRE("ana.py"),
-		FileName: true,
-	})
-
-	matches := sres.Files
-	if len(matches) != 1 || matches[0].FileName != "banana.py" {
-		t.Fatalf("got %v, want 1 match on 'banana.py'", matches)
-	}
-}
-
-func TestFileSubstringSearchBruteForce(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("BANZANA", []byte("x orange y"))
-	b.AddFile("banana", []byte("x apple y"))
-
-	q := &query.Substring{
-		Pattern:  "z",
-		FileName: true,
-	}
-
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 || res.Files[0].FileName != "BANZANA" {
-		t.Fatalf("got %v, want 1 match on 'BANZANA''", res.Files)
-	}
-}
-
-func TestFileSubstringSearchBruteForceEnd(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("BANZANA", []byte("x orange y"))
-	b.AddFile("bananaq", []byte("x apple y"))
-
-	q := &query.Substring{
-		Pattern:  "q",
-		FileName: true,
-	}
-
-	res := searchForTest(t, b, q)
-	if want := "bananaq"; len(res.Files) != 1 || res.Files[0].FileName != want {
-		t.Fatalf("got %v, want 1 match in %q", res.Files, want)
-	}
-}
-
-func TestSearchMatchAll(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("banzana", []byte("x orange y"))
-	// --------------------------0123456879
-	b.AddFile("banana", []byte("x apple y"))
-	sres := searchForTest(t, b, &query.Const{Value: true})
-
-	matches := sres.Files
-	if len(matches) != 2 {
-		t.Fatalf("got %v, want 2 matches", matches)
-	}
-}
-
-func TestSearchNewline(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("banzana", []byte("abcd\ndefg"))
-	sres := searchForTest(t, b, &query.Substring{Pattern: "d\nd"})
-
-	// Just check that we don't crash.
-
-	matches := sres.Files
-	if len(matches) != 1 {
-		t.Fatalf("got %v, want 1 matches", matches)
-	}
-}
-
-func TestSearchMatchAllRegexp(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("banzana", []byte("abcd"))
-	// --------------------------0123456879
-	b.AddFile("banana", []byte("pqrs"))
-	sres := searchForTest(t, b, &query.Regexp{Regexp: mustParseRE(".")})
-
-	matches := sres.Files
-	if len(matches) != 2 || sres.Stats.MatchCount != 2 {
-		t.Fatalf("got %v, want 2 matches", matches)
-	}
-	if len(matches[0].LineMatches[0].Line) != 4 || len(matches[1].LineMatches[0].Line) != 4 {
-		t.Fatalf("want 4 chars in every file, got %#v", matches)
-	}
-}
-
-func TestFileRestriction(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	b.AddFile("banana1", []byte("x orange y"))
-	// --------------------------0123456879
-	b.AddFile("banana2", []byte("x apple y"))
-	b.AddFile("orange", []byte("x apple y"))
-	sres := searchForTest(t, b, query.NewAnd(
-		&query.Substring{
-			Pattern:  "banana",
-			FileName: true,
-		},
-		&query.Substring{
-			Pattern: "apple",
-		}))
-
-	matches := sres.Files
-	if len(matches) != 1 || len(matches[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 match", matches)
-	}
-
-	match := matches[0].LineMatches[0]
-	got := string(match.Line)
-	want := "x apple y"
-	if got != want {
-		t.Errorf("got match %#v, want line %q", match, want)
-	}
-}
-
-func TestFileNameBoundary(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{Name: "banana2", Content: []byte("x apple y")},
-		Document{Name: "helpers.go", Content: []byte("x apple y")},
-		Document{Name: "foo", Content: []byte("x apple y")})
-	sres := searchForTest(t, b, &query.Substring{
-		Pattern:  "helpers.go",
-		FileName: true,
-	})
-
-	matches := sres.Files
-	if len(matches) != 1 || len(matches[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 match", matches)
-	}
-}
-
-func TestWordBoundaryRanking(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: []byte("xbytex xbytex")},
-		Document{Name: "f2", Content: []byte("xbytex\nbytex\nbyte bla")},
-		// -----------------------------------0123456 789012 34567890
-		Document{Name: "f3", Content: []byte("xbytex ybytex")})
-
-	sres := searchForTest(t, b, &query.Substring{
-		Pattern: "byte",
-	})
-
-	if len(sres.Files) != 3 {
-		t.Fatalf("got %#v, want 3 files", sres.Files)
-	}
-
-	file0 := sres.Files[0]
-	if file0.FileName != "f2" || len(file0.LineMatches) != 3 {
-		t.Fatalf("got file %s, num matches %d (%#v), want 3 matches in file f2", file0.FileName, len(file0.LineMatches), file0)
-	}
-
-	if file0.LineMatches[0].LineFragments[0].Offset != 13 {
-		t.Fatalf("got first match %#v, want full word match", sres.Files[0].LineMatches[0])
-	}
-	if file0.LineMatches[1].LineFragments[0].Offset != 7 {
-		t.Fatalf("got second match %#v, want partial word match", sres.Files[0].LineMatches[1])
-	}
-}
-
-func TestDocumentOrder(t *testing.T) {
-	var docs []Document
-	for i := 0; i < 3; i++ {
-		docs = append(docs, Document{Name: fmt.Sprintf("f%d", i), Content: []byte("needle")})
-	}
-
-	b := testIndexBuilder(t, nil, docs...)
-
-	sres := searchForTest(t, b, query.NewAnd(
-		&query.Substring{
-			Pattern: "needle",
-		}))
-
-	want := []string{"f0", "f1", "f2"}
-	var got []string
-	for _, f := range sres.Files {
-		got = append(got, f.FileName)
-	}
-	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("got %v, want %v", got, want)
-	}
-}
-
-func TestBranchMask(t *testing.T) {
-	b := testIndexBuilder(t, &Repository{
-		Branches: []RepositoryBranch{
-			{"master", "v-master"},
-			{"stable", "v-stable"},
-			{"bonzai", "v-bonzai"},
-		},
-	}, Document{Name: "f1", Content: []byte("needle"), Branches: []string{"master"}},
-		Document{Name: "f2", Content: []byte("needle"), Branches: []string{"stable", "master"}},
-		Document{Name: "f3", Content: []byte("needle"), Branches: []string{"stable", "master"}},
-		Document{Name: "f4", Content: []byte("needle"), Branches: []string{"bonzai"}},
-	)
-
-	sres := searchForTest(t, b, query.NewAnd(
-		&query.Substring{
-			Pattern: "needle",
-		},
-		&query.Branch{
-			Pattern: "table",
-		}))
-
-	if len(sres.Files) != 2 || sres.Files[0].FileName != "f2" || sres.Files[1].FileName != "f3" {
-		t.Fatalf("got %v, want 2 result from [f2,f3]", sres.Files)
-	}
-
-	if len(sres.Files[0].Branches) != 1 || sres.Files[0].Branches[0] != "stable" {
-		t.Fatalf("got %v, want 1 branch 'stable'", sres.Files[0].Branches)
-	}
-}
-
-func TestBranchLimit(t *testing.T) {
-	for limit := 64; limit <= 65; limit++ {
-		r := &Repository{}
-		for i := 0; i < limit; i++ {
-			s := fmt.Sprintf("b%d", i)
-			r.Branches = append(r.Branches, RepositoryBranch{
-				s, "v-" + s,
-			})
-		}
-		_, err := NewIndexBuilder(r)
-		if limit == 64 && err != nil {
-			t.Fatalf("NewIndexBuilder: %v", err)
-		} else if limit == 65 && err == nil {
-			t.Fatalf("NewIndexBuilder succeeded")
-		}
-	}
-}
-
-func TestBranchReport(t *testing.T) {
-	branches := []string{"stable", "master"}
-	b := testIndexBuilder(t, &Repository{
-		Branches: []RepositoryBranch{
-			{"stable", "vs"},
-			{"master", "vm"},
-		},
-	},
-		Document{Name: "f2", Content: []byte("needle"), Branches: branches})
-	sres := searchForTest(t, b, &query.Substring{
-		Pattern: "needle",
-	})
-	if len(sres.Files) != 1 {
-		t.Fatalf("got %v, want 1 result from f2", sres.Files)
-	}
-
-	f := sres.Files[0]
-	if !reflect.DeepEqual(f.Branches, branches) {
-		t.Fatalf("got branches %q, want %q", f.Branches, branches)
-	}
-}
-
-func TestBranchVersions(t *testing.T) {
-	b := testIndexBuilder(t, &Repository{
-		Branches: []RepositoryBranch{
-			{"stable", "v-stable"},
-			{"master", "v-master"},
-		},
-	}, Document{Name: "f2", Content: []byte("needle"), Branches: []string{"master"}})
-
-	sres := searchForTest(t, b, &query.Substring{
-		Pattern: "needle",
-	})
-	if len(sres.Files) != 1 {
-		t.Fatalf("got %v, want 1 result from f2", sres.Files)
-	}
-
-	f := sres.Files[0]
-	if f.Version != "v-master" {
-		t.Fatalf("got file %#v, want version 'v-master'", f)
-	}
-}
-
-func mustParseRE(s string) *syntax.Regexp {
-	r, err := syntax.Parse(s, 0)
-	if err != nil {
-		panic(err)
-	}
-
-	return r
-}
-
-func TestRegexp(t *testing.T) {
-	content := []byte("needle the bla")
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-		})
-	// ------------------------------01234567890123
-
-	sres := searchForTest(t, b,
-		&query.Regexp{
-			Regexp: mustParseRE("dle.*bla"),
-		})
-
-	if len(sres.Files) != 1 || len(sres.Files[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 match in 1 file", sres.Files)
-	}
-
-	got := sres.Files[0].LineMatches[0]
-	want := LineMatch{
-		LineFragments: []LineFragmentMatch{{
-			LineOffset:  3,
-			Offset:      3,
-			MatchLength: 11,
-		}},
-		Line:       content,
-		FileName:   false,
-		LineNumber: 1,
-		LineStart:  0,
-		LineEnd:    14,
-	}
-
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got %#v, want %#v", got, want)
-	}
-}
-
-func TestRegexpFile(t *testing.T) {
-	content := []byte("needle the bla")
-	// ----------------01234567890123
-
-	name := "let's play: find the mussel"
-	b := testIndexBuilder(t, nil,
-		Document{Name: name, Content: content},
-		Document{Name: "play.txt", Content: content})
-
-	sres := searchForTest(t, b,
-		&query.Regexp{
-			Regexp:   mustParseRE("play.*mussel"),
-			FileName: true,
-		})
-
-	if len(sres.Files) != 1 || len(sres.Files[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 match in 1 file", sres.Files)
-	}
-
-	if sres.Files[0].FileName != name {
-		t.Errorf("got match %#v, want name %q", sres.Files[0], name)
-	}
-}
-
-func TestRegexpOrder(t *testing.T) {
-	content := []byte("bla the needle")
-	// ----------------01234567890123
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-
-	sres := searchForTest(t, b,
-		&query.Regexp{
-			Regexp: mustParseRE("dle.*bla"),
-		})
-
-	if len(sres.Files) != 0 {
-		t.Fatalf("got %v, want 0 matches", sres.Files)
-	}
-}
-
-func TestRepoName(t *testing.T) {
-	content := []byte("bla the needle")
-	// ----------------01234567890123
-	b := testIndexBuilder(t, &Repository{Name: "bla"},
-		Document{Name: "f1", Content: content})
-
-	sres := searchForTest(t, b,
-		query.NewAnd(
-			&query.Substring{Pattern: "needle"},
-			&query.Repo{Pattern: "foo"},
-		))
-
-	if len(sres.Files) != 0 {
-		t.Fatalf("got %v, want 0 matches", sres.Files)
-	}
-
-	if sres.Stats.FilesConsidered > 0 {
-		t.Fatalf("got FilesConsidered %d, should have short circuited", sres.Stats.FilesConsidered)
-	}
-
-	sres = searchForTest(t, b,
-		query.NewAnd(
-			&query.Substring{Pattern: "needle"},
-			&query.Repo{Pattern: "bla"},
-		))
-	if len(sres.Files) != 1 {
-		t.Fatalf("got %v, want 1 match", sres.Files)
-	}
-}
-
-func TestMergeMatches(t *testing.T) {
-	content := []byte("blablabla")
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-
-	sres := searchForTest(t, b,
-		&query.Substring{Pattern: "bla"})
-	if len(sres.Files) != 1 || len(sres.Files[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 match", sres.Files)
-	}
-}
-
-func TestRepoURL(t *testing.T) {
-	content := []byte("blablabla")
-	b := testIndexBuilder(t, &Repository{
-		Name:                 "name",
-		URL:                  "URL",
-		CommitURLTemplate:    "commit",
-		FileURLTemplate:      "file-url",
-		LineFragmentTemplate: "fragment",
-	}, Document{Name: "f1", Content: content})
-
-	sres := searchForTest(t, b, &query.Substring{Pattern: "bla"})
-
-	if sres.RepoURLs["name"] != "file-url" {
-		t.Errorf("got RepoURLs %v, want {name: URL}", sres.RepoURLs)
-	}
-	if sres.LineFragments["name"] != "fragment" {
-		t.Errorf("got URLs %v, want {name: URL}", sres.LineFragments)
-	}
-}
-
-func TestRegexpCaseSensitive(t *testing.T) {
-	content := []byte("bla\nfunc unmarshalGitiles\n")
-	b := testIndexBuilder(t, nil, Document{
-		Name:    "f1",
-		Content: content,
-	})
-
-	res := searchForTest(t, b,
-		&query.Regexp{
-			Regexp:        mustParseRE("func.*Gitiles"),
-			CaseSensitive: true,
-		})
-
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, want one match", res.Files)
-	}
-}
-
-func TestRegexpCaseFolding(t *testing.T) {
-	content := []byte("bla\nfunc unmarshalGitiles\n")
-
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-	res := searchForTest(t, b,
-		&query.Regexp{
-			Regexp:        mustParseRE("func.*GITILES"),
-			CaseSensitive: false,
-		})
-
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, want one match", res.Files)
-	}
-}
-
-func TestCaseRegexp(t *testing.T) {
-	content := []byte("BLABLABLA")
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-	res := searchForTest(t, b,
-		&query.Regexp{
-			Regexp:        mustParseRE("[xb][xl][xa]"),
-			CaseSensitive: true,
-		})
-
-	if len(res.Files) > 0 {
-		t.Fatalf("got %v, want no matches", res.Files)
-	}
-}
-
-func TestNegativeRegexp(t *testing.T) {
-	content := []byte("BLABLABLA needle bla")
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-	res := searchForTest(t, b,
-		query.NewAnd(
-			&query.Substring{
-				Pattern: "needle",
-			},
-			&query.Not{
-				Child: &query.Regexp{
-					Regexp: mustParseRE(".cs"),
-				},
-			}))
-
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, want 1 match", res.Files)
-	}
-}
-
-func TestSymbolRank(t *testing.T) {
-	content := []byte("func bla() blubxxxxx")
-	// ----------------01234567890123456789
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-		}, Document{
-			Name:    "f2",
-			Content: content,
-			Symbols: []DocumentSection{{5, 8}},
-		}, Document{
-			Name:    "f3",
-			Content: content,
-		})
-
-	res := searchForTest(t, b,
-		&query.Substring{
-			CaseSensitive: false,
-			Pattern:       "bla",
-		})
-
-	if len(res.Files) != 3 {
-		t.Fatalf("got %d files, want 3 files. Full data: %v", len(res.Files), res.Files)
-	}
-	if res.Files[0].FileName != "f2" {
-		t.Errorf("got %#v, want 'f2' as top match", res.Files[0])
-	}
-}
-
-func TestSymbolRankRegexpUTF8(t *testing.T) {
-	prefix := strings.Repeat(string([]rune{kelvinCodePoint}), 100) + "\n"
-	content := []byte(prefix +
-		"func bla() blub")
-	// ------012345678901234
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-		}, Document{
-			Name:    "f2",
-			Content: content,
-			Symbols: []DocumentSection{{uint32(len(prefix) + 5), uint32(len(prefix) + 8)}},
-		}, Document{
-			Name:    "f3",
-			Content: content,
-		})
-
-	res := searchForTest(t, b,
-		&query.Regexp{
-			Regexp: mustParseRE("b.a"),
-		})
-
-	if len(res.Files) != 3 {
-		t.Fatalf("got %#v, want 3 files", res.Files)
-	}
-	if res.Files[0].FileName != "f2" {
-		t.Errorf("got %#v, want 'f2' as top match", res.Files[0])
-	}
-}
-
-func TestPartialSymbolRank(t *testing.T) {
-	content := []byte("func bla() blub")
-	// ----------------012345678901234
-
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-			Symbols: []DocumentSection{{4, 9}},
-		}, Document{
-			Name:    "f2",
-			Content: content,
-			Symbols: []DocumentSection{{4, 8}},
-		}, Document{
-			Name:    "f3",
-			Content: content,
-			Symbols: []DocumentSection{{4, 9}},
-		})
-
-	res := searchForTest(t, b,
-		&query.Substring{
-			Pattern: "bla",
-		})
-
-	if len(res.Files) != 3 {
-		t.Fatalf("got %#v, want 3 files", res.Files)
-	}
-	if res.Files[0].FileName != "f2" {
-		t.Errorf("got %#v, want 'f2' as top match", res.Files[0])
-	}
-}
-
-func TestNegativeRepo(t *testing.T) {
-	content := []byte("bla the needle")
-	// ----------------01234567890123
-	b := testIndexBuilder(t, &Repository{
-		Name: "bla",
-	}, Document{Name: "f1", Content: content})
-
-	sres := searchForTest(t, b,
-		query.NewAnd(
-			&query.Substring{Pattern: "needle"},
-			&query.Not{Child: &query.Repo{Pattern: "bla"}},
-		))
-
-	if len(sres.Files) != 0 {
-		t.Fatalf("got %v, want 0 matches", sres.Files)
-	}
-}
-
-func TestListRepos(t *testing.T) {
-	content := []byte("bla the needle")
-	// ----------------01234567890123
-	b := testIndexBuilder(t, &Repository{
-		Name: "reponame",
-	},
-		Document{Name: "f1", Content: content},
-		Document{Name: "f2", Content: content})
-
-	searcher := searcherForTest(t, b)
-	q := &query.Repo{Pattern: "epo"}
-	res, err := searcher.List(context.Background(), q)
-	if err != nil {
-		t.Fatalf("List(%v): %v", q, err)
-	}
-	if len(res.Repos) != 1 || res.Repos[0].Repository.Name != "reponame" {
-		t.Fatalf("got %v, want 1 matches", res)
-	}
-	if got := res.Repos[0].Stats.Shards; got != 1 {
-		t.Fatalf("got %d, want 1 shard", got)
-	}
-	q = &query.Repo{Pattern: "bla"}
-	res, err = searcher.List(context.Background(), q)
-	if err != nil {
-		t.Fatalf("List(%v): %v", q, err)
-	}
-	if len(res.Repos) != 0 {
-		t.Fatalf("got %v, want 0 matches", res)
-	}
-}
-
-func TestMetadata(t *testing.T) {
-	content := []byte("bla the needle")
-	// ----------------01234567890123
-	b := testIndexBuilder(t, &Repository{
-		Name: "reponame",
-	}, Document{Name: "f1", Content: content},
-		Document{Name: "f2", Content: content})
-
-	var buf bytes.Buffer
-	b.Write(&buf)
-	f := &memSeeker{buf.Bytes()}
-
-	rd, _, err := ReadMetadata(f)
-	if err != nil {
-		t.Fatalf("ReadMetadata: %v", err)
-	}
-
-	if got, want := rd.Name, "reponame"; got != want {
-		t.Fatalf("got %q want %q", got, want)
-	}
-}
-
-func TestOr(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: []byte("needle")},
-		Document{Name: "f2", Content: []byte("banana")})
-	sres := searchForTest(t, b, query.NewOr(
-		&query.Substring{Pattern: "needle"},
-		&query.Substring{Pattern: "banana"}))
-
-	if len(sres.Files) != 2 {
-		t.Fatalf("got %v, want 2 files", sres.Files)
-	}
-}
-
-func TestAtomCountScore(t *testing.T) {
-	b := testIndexBuilder(t,
-		&Repository{
-			Branches: []RepositoryBranch{
-				{"branches", "v1"},
-				{"needle", "v2"},
-			},
-		},
-		Document{Name: "f1", Content: []byte("needle the bla"), Branches: []string{"branches"}},
-		Document{Name: "needle-file-branch", Content: []byte("needle content"), Branches: []string{"needle"}},
-		Document{Name: "needle-file", Content: []byte("needle content"), Branches: []string{"branches"}})
-
-	sres := searchForTest(t, b,
-		query.NewOr(
-			&query.Substring{Pattern: "needle"},
-			&query.Substring{Pattern: "needle", FileName: true},
-			&query.Branch{Pattern: "needle"},
-		))
-	var got []string
-	for _, f := range sres.Files {
-		got = append(got, f.FileName)
-	}
-	want := []string{"needle-file-branch", "needle-file", "f1"}
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestImportantCutoff(t *testing.T) {
-	content := []byte("func bla() blub")
-	// ----------------012345678901234
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-			Symbols: []DocumentSection{{5, 8}},
-		}, Document{
-			Name:    "f2",
-			Content: content,
-		})
-	opts := SearchOptions{
-		ShardMaxImportantMatch: 1,
-	}
-
-	sres := searchForTest(t, b, &query.Substring{Pattern: "bla"}, opts)
-	if len(sres.Files) != 1 || sres.Files[0].FileName != "f1" {
-		t.Errorf("got %v, wanted 1 match 'f1'", sres.Files)
-	}
-}
-
-func TestFrequency(t *testing.T) {
-	content := []byte("sla _Py_HashDouble(double v sla las las shd dot dot")
-	// ----------------012345678901234
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-		})
-
-	sres := searchForTest(t, b, &query.Substring{Pattern: "slashdot"})
-	if len(sres.Files) != 0 {
-		t.Errorf("got %v, wanted 0 matches", sres.Files)
-	}
-}
-
-func TestMatchNewline(t *testing.T) {
-	re, err := syntax.Parse("[^a]a", syntax.ClassNL)
-	if err != nil {
-		t.Fatalf("syntax.Parse: %v", err)
-	}
-
-	content := []byte("pqr\nalex")
-	// ----------------0123 4567
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-		})
-
-	sres := searchForTest(t, b, &query.Regexp{Regexp: re, CaseSensitive: true})
-	if len(sres.Files) != 1 {
-		t.Errorf("got %v, wanted 1 matches", sres.Files)
-	} else if l := sres.Files[0].LineMatches[0].Line; !bytes.Equal(l, content[len("pqr\n"):]) {
-		t.Errorf("got match line %q, want %q", l, content)
-	}
-}
-
-func TestSubRepo(t *testing.T) {
-	subRepos := map[string]*Repository{
-		"sub": {
-			Name:                 "sub-name",
-			LineFragmentTemplate: "sub-line",
-		},
-	}
-
-	content := []byte("pqr\nalex")
-	// ----------------0123 4567
-
-	b := testIndexBuilder(t, &Repository{
-		SubRepoMap: subRepos,
-	}, Document{
-		Name:              "sub/f1",
-		Content:           content,
-		SubRepositoryPath: "sub",
-	})
-
-	sres := searchForTest(t, b, &query.Substring{Pattern: "alex"})
-	if len(sres.Files) != 1 {
-		t.Fatalf("got %v, wanted 1 matches", sres.Files)
-	}
-
-	f := sres.Files[0]
-	if f.SubRepositoryPath != "sub" || f.SubRepositoryName != "sub-name" {
-		t.Errorf("got %#v, want SubRepository{Path,Name} = {'sub', 'sub-name'}", f)
-	}
-
-	if sres.LineFragments["sub-name"] != "sub-line" {
-		t.Errorf("got LineFragmentTemplate %v, want {'sub':'sub-line'}", sres.LineFragments)
-	}
-}
-
-func TestSearchEither(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: []byte("bla needle bla")},
-		Document{Name: "needle-file-branch", Content: []byte("bla content")})
-
-	sres := searchForTest(t, b, &query.Substring{Pattern: "needle"})
-	if len(sres.Files) != 2 {
-		t.Fatalf("got %v, wanted 2 matches", sres.Files)
-	}
-
-	sres = searchForTest(t, b, &query.Substring{Pattern: "needle", Content: true})
-	if len(sres.Files) != 1 {
-		t.Fatalf("got %v, wanted 1 match", sres.Files)
-	}
-
-	if got, want := sres.Files[0].FileName, "f1"; got != want {
-		t.Errorf("got %q, want %q", got, want)
-	}
-}
-
-func TestUnicodeExactMatch(t *testing.T) {
-	needle := "néédlÉ"
-	content := []byte("blá blá " + needle + " blâ")
-	// ----------------01234567    8
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-
-	if res := searchForTest(t, b, &query.Substring{Pattern: needle, CaseSensitive: true}); len(res.Files) != 1 {
-		t.Fatalf("case sensitive: got %v, wanted 1 match", res.Files)
-	}
-}
-
-func TestUnicodeCoverContent(t *testing.T) {
-	needle := "néédlÉ"
-	content := []byte("blá blá " + needle + " blâ")
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-
-	if res := searchForTest(t, b, &query.Substring{Pattern: "NÉÉDLÉ", CaseSensitive: true}); len(res.Files) != 0 {
-		t.Fatalf("case sensitive: got %v, wanted 0 match", res.Files)
-	}
-
-	res := searchForTest(t, b, &query.Substring{Pattern: "NÉÉDLÉ"})
-	if len(res.Files) != 1 {
-		t.Fatalf("case insensitive: got %v, wanted 1 match", res.Files)
-	}
-
-	if got, want := res.Files[0].LineMatches[0].LineFragments[0].Offset, uint32(strings.Index(string(content), needle)); got != want {
-		t.Errorf("got %d want %d", got, want)
-	}
-}
-
-func TestUnicodeNonCoverContent(t *testing.T) {
-	needle := "nééáádlÉ"
-	//---------01234567
-	content := []byte("blá blá " + needle + " blâ")
-	// ----------------01234567    8901234   5678
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: content})
-
-	res := searchForTest(t, b, &query.Substring{Pattern: "NÉÉÁÁDLÉ", Content: true})
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, wanted 1 match", res.Files)
-	}
-
-	if got, want := res.Files[0].LineMatches[0].LineFragments[0].Offset, uint32(strings.Index(string(content), needle)); got != want {
-		t.Errorf("got %d want %d", got, want)
-	}
-}
-
-const kelvinCodePoint = 8490
-
-func TestUnicodeVariableLength(t *testing.T) {
-	lower := 'k'
-	upper := rune(kelvinCodePoint)
-
-	needle := "nee" + string([]rune{lower}) + "eed"
-	corpus := []byte("nee" + string([]rune{upper}) + "eed" +
-		" ee" + string([]rune{lower}) + "ee" +
-		" ee" + string([]rune{upper}) + "ee")
-
-	b := testIndexBuilder(t, nil,
-		Document{Name: "f1", Content: []byte(corpus)})
-
-	res := searchForTest(t, b, &query.Substring{Pattern: needle, Content: true})
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, wanted 1 match", res.Files)
-	}
-}
-
-func TestUnicodeFileStartOffsets(t *testing.T) {
-	unicode := "世界"
-	wat := "waaaaaat"
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: []byte(unicode),
-		},
-		Document{
-			Name:    "f2",
-			Content: []byte(wat),
-		},
-	)
-	q := &query.Substring{Pattern: wat, Content: true}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, wanted 1 match", res.Files)
-	}
-}
-
-func TestLongFileUTF8(t *testing.T) {
-	needle := "neeedle"
-
-	// 6 bytes.
-	unicode := "世界"
-	content := []byte(strings.Repeat(unicode, 100) + needle)
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: []byte(strings.Repeat("a", 50)),
-		},
-		Document{
-			Name:    "f2",
-			Content: content,
-		})
-
-	q := &query.Substring{Pattern: needle, Content: true}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Errorf("got %v, want 1 result", res)
-	}
-}
-
-func TestEstimateDocCount(t *testing.T) {
-	content := []byte("bla needle bla")
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: content},
-		Document{Name: "f2", Content: content},
-	)
-
-	if sres := searchForTest(t, b,
-		query.NewAnd(
-			&query.Substring{Pattern: "needle"},
-			&query.Repo{Pattern: "reponame"},
-		), SearchOptions{
-			EstimateDocCount: true,
-		}); sres.Stats.ShardFilesConsidered != 2 {
-		t.Errorf("got FilesConsidered = %d, want 2", sres.Stats.FilesConsidered)
-	}
-	if sres := searchForTest(t, b,
-		query.NewAnd(
-			&query.Substring{Pattern: "needle"},
-			&query.Repo{Pattern: "nomatch"},
-		), SearchOptions{
-			EstimateDocCount: true,
-		}); sres.Stats.ShardFilesConsidered != 0 {
-		t.Errorf("got FilesConsidered = %d, want 0", sres.Stats.FilesConsidered)
-	}
-}
-
-func TestUTF8CorrectCorpus(t *testing.T) {
-	needle := "neeedle"
-
-	// 6 bytes.
-	unicode := "世界"
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: []byte(strings.Repeat(unicode, 100)),
-		},
-		Document{
-			Name:    "xxxxxneeedle",
-			Content: []byte("hello"),
-		})
-
-	q := &query.Substring{Pattern: needle, FileName: true}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Errorf("got %v, want 1 result", res)
-	}
-}
-
-func TestBuilderStats(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: []byte(strings.Repeat("abcd", 1024)),
-		})
-	var buf bytes.Buffer
-	b.Write(&buf)
-
-	if got, want := b.ContentSize(), uint32(2+4*1024); got != want {
-		t.Errorf("got %d, want %d", got, want)
-	}
-}
-
-func TestIOStats(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: []byte(strings.Repeat("abcd", 1024)),
-		})
-
-	q := &query.Substring{Pattern: "abc", CaseSensitive: true, Content: true}
-	res := searchForTest(t, b, q)
-
-	// 4096 (content) + 2 (overhead: newlines or doc sections)
-	if got, want := res.Stats.ContentBytesLoaded, int64(4098); got != want {
-		t.Errorf("got content I/O %d, want %d", got, want)
-	}
-
-	// 1024 entries, each 4 bytes apart. 4 fits into single byte
-	// delta encoded.
-	if got, want := res.Stats.IndexBytesLoaded, int64(1024); got != want {
-		t.Errorf("got index I/O %d, want %d", got, want)
-	}
-}
-
-func TestStartLineAnchor(t *testing.T) {
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name: "f1",
-			Content: []byte(
-				`hello
-start of middle of line
-`),
-		})
-
-	q, err := query.Parse("^start")
-	if err != nil {
-		t.Errorf("parse: %v", err)
-	}
-
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Errorf("got %v, want 1 file", res.Files)
-	}
-
-	q, err = query.Parse("^middle")
-	if err != nil {
-		t.Errorf("parse: %v", err)
-	}
-	res = searchForTest(t, b, q)
-	if len(res.Files) != 0 {
-		t.Errorf("got %v, want 0 files", res.Files)
-	}
-}
-
-func TestAndOrUnicode(t *testing.T) {
-	q, err := query.Parse("orange.*apple")
-	if err != nil {
-		t.Errorf("parse: %v", err)
-	}
-	finalQ := query.NewAnd(q,
-		query.NewOr(query.NewAnd(&query.Repo{Pattern: "name"},
-			query.NewOr(&query.Branch{Pattern: "master"}))))
-
-	b := testIndexBuilder(t, &Repository{
-		Name:     "name",
-		Branches: []RepositoryBranch{{"master", "master-version"}},
-	}, Document{
-		Name:    "f2",
-		Content: []byte("orange\u2318apple"),
-		// --------------0123456     78901
-		Branches: []string{"master"},
-	})
-
-	res := searchForTest(t, b, finalQ)
-	if len(res.Files) != 1 {
-		t.Errorf("got %v, want 1 result", res.Files)
-	}
-}
-
-func TestAndShort(t *testing.T) {
-	content := []byte("bla needle at orange bla")
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: content},
-		Document{Name: "f2", Content: []byte("xx at xx")},
-		Document{Name: "f3", Content: []byte("yy orange xx")},
-	)
-
-	q := query.NewAnd(&query.Substring{Pattern: "at"},
-		&query.Substring{Pattern: "orange"})
-
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 || res.Files[0].FileName != "f1" {
-		t.Errorf("got %v, want 1 result", res.Files)
-	}
-}
-
-func TestNoCollectRegexpSubstring(t *testing.T) {
-	content := []byte("bla final bla\nfoo final, foo")
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: content},
-	)
-
-	q := &query.Regexp{
-		Regexp: mustParseRE("final[,.]"),
-	}
-
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, want 1 result", res.Files)
-	}
-	if f := res.Files[0]; len(f.LineMatches) != 1 {
-		t.Fatalf("got line matches %v, want 1 line match", printLineMatches(f.LineMatches))
-	}
-}
-
-func printLineMatches(ms []LineMatch) string {
-	var ss []string
-	for _, m := range ms {
-		ss = append(ss, fmt.Sprintf("%d:%q %v", m.LineNumber, m.Line, m.LineFragments))
-	}
-
-	return strings.Join(ss, ", ")
-}
-
-func TestLang(t *testing.T) {
-	content := []byte("bla needle bla")
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: content},
-		Document{Name: "f2", Language: "java", Content: content},
-		Document{Name: "f3", Language: "cpp", Content: content},
-	)
-
-	q := query.NewAnd(&query.Substring{Pattern: "needle"},
-		&query.Language{Language: "cpp"})
-
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, want 1 result in f3", res.Files)
-	}
-	f := res.Files[0]
-	if f.FileName != "f3" || f.Language != "cpp" {
-		t.Fatalf("got %v, want 1 match with language cpp", f)
-	}
-}
-
-func TestLangShortcut(t *testing.T) {
-	content := []byte("bla needle bla")
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f2", Language: "java", Content: content},
-		Document{Name: "f3", Language: "cpp", Content: content},
-	)
-
-	q := query.NewAnd(&query.Substring{Pattern: "needle"},
-		&query.Language{Language: "fortran"})
-
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 0 {
-		t.Fatalf("got %v, want 0 results", res.Files)
-	}
-	if res.Stats.IndexBytesLoaded > 0 {
-		t.Errorf("got IndexBytesLoaded %d, want 0", res.Stats.IndexBytesLoaded)
-	}
-}
-
-func TestNoTextMatchAtoms(t *testing.T) {
-	content := []byte("bla needle bla")
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: content},
-		Document{Name: "f2", Language: "java", Content: content},
-		Document{Name: "f3", Language: "cpp", Content: content},
-	)
-	q := query.NewAnd(&query.Language{Language: "java"})
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Fatalf("got %v, want 1 result in f3", res.Files)
-	}
-}
-
-func TestNoPositiveAtoms(t *testing.T) {
-	content := []byte("bla needle bla")
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: content},
-		Document{Name: "f2", Content: content},
-	)
-
-	q := query.NewAnd(
-		&query.Not{Child: &query.Substring{Pattern: "xyz"}},
-		&query.Repo{Pattern: "reponame"})
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 2 {
-		t.Fatalf("got %v, want 2 results in f3", res.Files)
-	}
-}
-
-func TestSymbolBoundaryStart(t *testing.T) {
-	content := []byte("start\nbla bla\nend")
-	// ----------------012345 67890123 456
-
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{
-			Name:    "f1",
-			Content: content,
-			Symbols: []DocumentSection{{0, 5}, {14, 17}},
-		},
-	)
-	q := &query.Symbol{
-		Atom: &query.Substring{Pattern: "start"},
-	}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 || len(res.Files[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 line in 1 file", res.Files)
-	}
-	m := res.Files[0].LineMatches[0].LineFragments[0]
-	if m.Offset != 0 {
-		t.Fatalf("got offset %d want 0", m.Offset)
-	}
-}
-
-func TestSymbolBoundaryEnd(t *testing.T) {
-	content := []byte("start\nbla bla\nend")
-	// ----------------012345 67890123 456
-
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{
-			Name:    "f1",
-			Content: content,
-			Symbols: []DocumentSection{{14, 17}},
-		},
-	)
-	q := &query.Symbol{
-		Atom: &query.Substring{Pattern: "end"},
-	}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 || len(res.Files[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 line in 1 file", res.Files)
-	}
-	m := res.Files[0].LineMatches[0].LineFragments[0]
-	if m.Offset != 14 {
-		t.Fatalf("got offset %d want 0", m.Offset)
-	}
-}
-
-func TestSymbolAtom(t *testing.T) {
-	content := []byte("bla\nsymblabla\nbla")
-	// ----------------0123 456789012
-
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{
-			Name:    "f1",
-			Content: content,
-			Symbols: []DocumentSection{{4, 12}},
-		},
-	)
-	q := &query.Symbol{
-		Atom: &query.Substring{Pattern: "bla"},
-	}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 || len(res.Files[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 line in 1 file", res.Files)
-	}
-	m := res.Files[0].LineMatches[0].LineFragments[0]
-	if m.Offset != 7 || m.MatchLength != 3 {
-		t.Fatalf("got offset %d, size %d want 7 size 3", m.Offset, m.MatchLength)
-	}
-}
-
-func TestSymbolAtomExact(t *testing.T) {
-	content := []byte("bla\nsym\nbla\nsym\nasymb")
-	// ----------------0123 4567 89012
-
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{
-			Name:    "f1",
-			Content: content,
-			Symbols: []DocumentSection{{4, 7}},
-		},
-	)
-	q := &query.Symbol{
-		Atom: &query.Substring{Pattern: "sym"},
-	}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 || len(res.Files[0].LineMatches) != 1 {
-		t.Fatalf("got %v, want 1 line in 1 file", res.Files)
-	}
-	m := res.Files[0].LineMatches[0].LineFragments[0]
-	if m.Offset != 4 {
-		t.Fatalf("got offset %d, want 7", m.Offset)
-	}
-}
-
-func TestHitIterTerminate(t *testing.T) {
-	// contrived input: trigram frequencies forces selecting abc +
-	// def for the distance iteration. There is no match, so this
-	// will advance the compressedPostingIterator to beyond the
-	// end.
-	content := []byte("abc bcdbcd cdecde abcabc def efg")
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-		},
-	)
-	searchForTest(t, b, &query.Substring{Pattern: "abcdef"})
-}
-
-func TestDistanceHitIterBailLast(t *testing.T) {
-	content := []byte("AST AST AST UASH")
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: content,
-		},
-	)
-	res := searchForTest(t, b, &query.Substring{Pattern: "UAST"})
-	if len(res.Files) != 0 {
-		t.Fatalf("got %v, want no results", res.Files)
-	}
-}
-
-func TestDocumentSectionRuneBoundary(t *testing.T) {
-	content := string([]rune{kelvinCodePoint, kelvinCodePoint, kelvinCodePoint})
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	for i, sec := range []DocumentSection{
-		{2, 6},
-		{3, 7},
-	} {
-		if err := b.Add(Document{
-			Name:    "f1",
-			Content: []byte(content),
-			Symbols: []DocumentSection{sec},
-		}); err == nil {
-			t.Errorf("%d: Add succeeded", i)
-		}
-	}
-}
-
-func TestUnicodeQuery(t *testing.T) {
-	content := string([]rune{kelvinCodePoint, kelvinCodePoint, kelvinCodePoint})
-	b := testIndexBuilder(t, nil,
-		Document{
-			Name:    "f1",
-			Content: []byte(content),
-		},
-	)
-
-	q := &query.Substring{Pattern: content}
-	res := searchForTest(t, b, q)
-	if len(res.Files) != 1 {
-		t.Fatalf("want 1 match, got %v", res.Files)
-	}
-
-	f := res.Files[0]
-	if len(f.LineMatches) != 1 {
-		t.Fatalf("want 1 line, got %v", f.LineMatches)
-	}
-	l := f.LineMatches[0]
-
-	if len(l.LineFragments) != 1 {
-		t.Fatalf("want 1 line fragment, got %v", l.LineFragments)
-	}
-	fr := l.LineFragments[0]
-	if fr.MatchLength != len(content) {
-		t.Fatalf("got MatchLength %d want %d", fr.MatchLength, len(content))
-	}
-}
-
-func TestSkipInvalidContent(t *testing.T) {
-	for _, content := range []string{
-		// Binary
-		"abc def \x00 abc",
-	} {
-
-		b, err := NewIndexBuilder(nil)
-		if err != nil {
-			t.Fatalf("NewIndexBuilder: %v", err)
-		}
-
-		if err := b.Add(Document{
-			Name:    "f1",
-			Content: []byte(content),
-		}); err != nil {
-			t.Fatal(err)
-		}
-
-		q := &query.Substring{Pattern: "abc def"}
-		res := searchForTest(t, b, q)
-		if len(res.Files) != 0 {
-			t.Fatalf("got %v, want no results", res.Files)
-		}
-
-		q = &query.Substring{Pattern: "NOT-INDEXED"}
-		res = searchForTest(t, b, q)
-		if len(res.Files) != 1 {
-			t.Fatalf("got %v, want 1 result", res.Files)
-		}
-	}
-}
-
-func TestCheckText(t *testing.T) {
-	for _, text := range []string{"", "simple ascii", "símplé unicödé", "\uFEFFwith utf8 'bom'", "with \uFFFD unicode replacement char"} {
-		if err := CheckText([]byte(text), 20000); err != nil {
-			t.Errorf("CheckText(%q): %v", text, err)
-		}
-	}
-	for _, text := range []string{"zero\x00byte", "xx", "0123456789abcdefghi"} {
-		if err := CheckText([]byte(text), 15); err == nil {
-			t.Errorf("CheckText(%q) succeeded", text)
-		}
-	}
-}
-
-func TestLineAnd(t *testing.T) {
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: []byte("apple\nbanana\napple banana chocolate apple pudding banana\ngrape")},
-		Document{Name: "f2", Content: []byte("apple orange\nbanana")},
-		Document{Name: "f3", Content: []byte("banana grape")},
-	)
-	pattern := "(apple)(?-s:.)*?(banana)"
-	r, _ := syntax.Parse(pattern, syntax.Perl)
-
-	q := query.Regexp{
-		Regexp:  r,
-		Content: true,
-	}
-	res := searchForTest(t, b, &q)
-	wantRegexpCount := 1
-	if gotRegexpCount := res.RegexpsConsidered; gotRegexpCount != wantRegexpCount {
-		t.Errorf("got %d, wanted %d", gotRegexpCount, wantRegexpCount)
-	}
-	if len(res.Files) != 1 || res.Files[0].FileName != "f1" {
-		t.Errorf("got %v, want 1 result", res.Files)
-	}
-}
-
-func TestLineAndFileName(t *testing.T) {
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: []byte("apple banana\ngrape")},
-		Document{Name: "f2", Content: []byte("apple banana\norange")},
-		Document{Name: "apple banana", Content: []byte("banana grape")},
-	)
-	pattern := "(apple)(?-s:.)*?(banana)"
-	r, _ := syntax.Parse(pattern, syntax.Perl)
-
-	q := query.Regexp{
-		Regexp:   r,
-		FileName: true,
-	}
-	res := searchForTest(t, b, &q)
-	wantRegexpCount := 1
-	if gotRegexpCount := res.RegexpsConsidered; gotRegexpCount != wantRegexpCount {
-		t.Errorf("got %d, wanted %d", gotRegexpCount, wantRegexpCount)
-	}
-	if len(res.Files) != 1 || res.Files[0].FileName != "apple banana" {
-		t.Errorf("got %v, want 1 result", res.Files)
-	}
-}
-
-func TestMultiLineRegex(t *testing.T) {
-	b := testIndexBuilder(t, &Repository{Name: "reponame"},
-		Document{Name: "f1", Content: []byte("apple banana\ngrape")},
-		Document{Name: "f2", Content: []byte("apple orange")},
-		Document{Name: "f3", Content: []byte("grape apple")},
-	)
-	pattern := "(apple).*?[[:space:]].*?(grape)"
-	r, _ := syntax.Parse(pattern, syntax.Perl)
-
-	q := query.Regexp{
-		Regexp: r,
-	}
-	res := searchForTest(t, b, &q)
-	wantRegexpCount := 2
-	if gotRegexpCount := res.RegexpsConsidered; gotRegexpCount != wantRegexpCount {
-		t.Errorf("got %d, wanted %d", gotRegexpCount, wantRegexpCount)
-	}
-	if len(res.Files) != 1 || res.Files[0].FileName != "f1" {
-		t.Errorf("got %v, want 1 result", res.Files)
-	}
-}
diff --git a/indexbuilder.go b/indexbuilder.go
deleted file mode 100644
index b5c87aa..0000000
--- a/indexbuilder.go
+++ /dev/null
@@ -1,414 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"hash/crc64"
-	"html/template"
-	"log"
-	"path/filepath"
-	"sort"
-	"unicode/utf8"
-)
-
-var _ = log.Println
-
-const ngramSize = 3
-
-type searchableString struct {
-	data []byte
-}
-
-// Filled by the linker (see build-deploy.sh)
-var Version string
-
-// Store character (unicode codepoint) offset (in bytes) this often.
-const runeOffsetFrequency = 100
-
-type postingsBuilder struct {
-	postings    map[ngram][]byte
-	lastOffsets map[ngram]uint32
-
-	// To support UTF-8 searching, we must map back runes to byte
-	// offsets. As a first attempt, we sample regularly. The
-	// precise offset can be found by walking from the recorded
-	// offset to the desired rune.
-	runeOffsets []uint32
-	runeCount   uint32
-
-	isPlainASCII bool
-
-	endRunes []uint32
-	endByte  uint32
-}
-
-func newPostingsBuilder() *postingsBuilder {
-	return &postingsBuilder{
-		postings:     map[ngram][]byte{},
-		lastOffsets:  map[ngram]uint32{},
-		isPlainASCII: true,
-	}
-}
-
-// Store trigram offsets for the given UTF-8 data. The
-// DocumentSections must correspond to rune boundaries in the UTF-8
-// data.
-func (s *postingsBuilder) newSearchableString(data []byte, byteSections []DocumentSection) (*searchableString, []DocumentSection, error) {
-	dest := searchableString{
-		data: data,
-	}
-	var buf [8]byte
-	var runeGram [3]rune
-
-	var runeIndex uint32
-	byteCount := 0
-	dataSz := uint32(len(data))
-
-	byteSectionBoundaries := make([]uint32, 0, 2*len(byteSections))
-	for _, s := range byteSections {
-		byteSectionBoundaries = append(byteSectionBoundaries, s.Start, s.End)
-	}
-	var runeSectionBoundaries []uint32
-
-	endRune := s.runeCount
-	for ; len(data) > 0; runeIndex++ {
-		c, sz := utf8.DecodeRune(data)
-		if sz > 1 {
-			s.isPlainASCII = false
-		}
-		data = data[sz:]
-
-		runeGram[0], runeGram[1], runeGram[2] = runeGram[1], runeGram[2], c
-
-		if idx := s.runeCount + runeIndex; idx%runeOffsetFrequency == 0 {
-			s.runeOffsets = append(s.runeOffsets, s.endByte+uint32(byteCount))
-		}
-		for len(byteSectionBoundaries) > 0 && byteSectionBoundaries[0] == uint32(byteCount) {
-			runeSectionBoundaries = append(runeSectionBoundaries,
-				endRune+uint32(runeIndex))
-			byteSectionBoundaries = byteSectionBoundaries[1:]
-		}
-
-		byteCount += sz
-
-		if runeIndex < 2 {
-			continue
-		}
-
-		ng := runesToNGram(runeGram)
-		lastOff := s.lastOffsets[ng]
-		newOff := endRune + uint32(runeIndex) - 2
-
-		m := binary.PutUvarint(buf[:], uint64(newOff-lastOff))
-		s.postings[ng] = append(s.postings[ng], buf[:m]...)
-		s.lastOffsets[ng] = newOff
-	}
-	s.runeCount += runeIndex
-
-	for len(byteSectionBoundaries) > 0 && byteSectionBoundaries[0] < uint32(byteCount) {
-		return nil, nil, fmt.Errorf("no rune for section boundary at byte %d", byteSectionBoundaries[0])
-	}
-
-	// Handle symbol definition that ends at file end. This can
-	// happen for labels at the end of .bat files.
-
-	for len(byteSectionBoundaries) > 0 && byteSectionBoundaries[0] == uint32(byteCount) {
-		runeSectionBoundaries = append(runeSectionBoundaries,
-			endRune+runeIndex)
-		byteSectionBoundaries = byteSectionBoundaries[1:]
-	}
-	runeSecs := make([]DocumentSection, 0, len(byteSections))
-	for i := 0; i < len(runeSectionBoundaries); i += 2 {
-		runeSecs = append(runeSecs, DocumentSection{
-			Start: runeSectionBoundaries[i],
-			End:   runeSectionBoundaries[i+1],
-		})
-	}
-
-	s.endRunes = append(s.endRunes, s.runeCount)
-	s.endByte += dataSz
-	return &dest, runeSecs, nil
-}
-
-// IndexBuilder builds a single index shard.
-type IndexBuilder struct {
-	contentStrings  []*searchableString
-	nameStrings     []*searchableString
-	docSections     [][]DocumentSection
-	runeDocSections []DocumentSection
-
-	checksums []byte
-
-	branchMasks []uint64
-	subRepos    []uint32
-
-	contentPostings *postingsBuilder
-	namePostings    *postingsBuilder
-
-	// root repository
-	repo Repository
-
-	// name to index.
-	subRepoIndices map[string]uint32
-
-	// language => language code
-	languageMap map[string]byte
-
-	// languages codes
-	languages []byte
-}
-
-func (d *Repository) verify() error {
-	for _, t := range []string{d.FileURLTemplate, d.LineFragmentTemplate, d.CommitURLTemplate} {
-		if _, err := template.New("").Parse(t); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// ContentSize returns the number of content bytes so far ingested.
-func (b *IndexBuilder) ContentSize() uint32 {
-	// Add the name too so we don't skip building index if we have
-	// lots of empty files.
-	return b.contentPostings.endByte + b.namePostings.endByte
-}
-
-// NewIndexBuilder creates a fresh IndexBuilder. The passed in
-// Repository contains repo metadata, and may be set to nil.
-func NewIndexBuilder(r *Repository) (*IndexBuilder, error) {
-	b := &IndexBuilder{
-		contentPostings: newPostingsBuilder(),
-		namePostings:    newPostingsBuilder(),
-		languageMap:     map[string]byte{},
-	}
-
-	if r == nil {
-		r = &Repository{}
-	}
-	if err := b.setRepository(r); err != nil {
-		return nil, err
-	}
-	return b, nil
-}
-
-func (b *IndexBuilder) setRepository(desc *Repository) error {
-	if len(b.contentStrings) > 0 {
-		return fmt.Errorf("setRepository called after adding files")
-	}
-	if err := desc.verify(); err != nil {
-		return err
-	}
-
-	if len(desc.Branches) > 64 {
-		return fmt.Errorf("too many branches")
-	}
-
-	b.repo = *desc
-
-	// copy subrepomap without root
-	b.repo.SubRepoMap = map[string]*Repository{}
-	for k, v := range desc.SubRepoMap {
-		if k != "" {
-			b.repo.SubRepoMap[k] = v
-		}
-	}
-
-	b.populateSubRepoIndices()
-	return nil
-}
-
-type DocumentSection struct {
-	Start, End uint32
-}
-
-// Document holds a document (file) to index.
-type Document struct {
-	Name              string
-	Content           []byte
-	Branches          []string
-	SubRepositoryPath string
-	Language          string
-
-	// If set, something is wrong with the file contents, and this
-	// is the reason it wasn't indexed.
-	SkipReason string
-
-	// Document sections for symbols. Offsets should use bytes.
-	Symbols []DocumentSection
-}
-
-type docSectionSlice []DocumentSection
-
-func (m docSectionSlice) Len() int           { return len(m) }
-func (m docSectionSlice) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
-func (m docSectionSlice) Less(i, j int) bool { return m[i].Start < m[j].Start }
-
-// AddFile is a convenience wrapper for Add
-func (b *IndexBuilder) AddFile(name string, content []byte) error {
-	return b.Add(Document{Name: name, Content: content})
-}
-
-// CheckText returns a reason why the given contents are probably not source texts.
-func CheckText(content []byte, maxTrigramCount int) error {
-	if len(content) == 0 {
-		return nil
-	}
-
-	if len(content) < ngramSize {
-		return fmt.Errorf("file size smaller than %d", ngramSize)
-	}
-
-	trigrams := map[ngram]struct{}{}
-
-	var cur [3]rune
-	byteCount := 0
-	for len(content) > 0 {
-		if content[0] == 0 {
-			return fmt.Errorf("binary data at byte offset %d", byteCount)
-		}
-
-		r, sz := utf8.DecodeRune(content)
-		content = content[sz:]
-		byteCount += sz
-
-		cur[0], cur[1], cur[2] = cur[1], cur[2], r
-		if cur[0] == 0 {
-			// start of file.
-			continue
-		}
-
-		trigrams[runesToNGram(cur)] = struct{}{}
-		if len(trigrams) > maxTrigramCount {
-			// probably not text.
-			return fmt.Errorf("number of trigrams exceeds %d", maxTrigramCount)
-		}
-	}
-	return nil
-}
-
-func (b *IndexBuilder) populateSubRepoIndices() {
-	if b.subRepoIndices != nil {
-		return
-	}
-	paths := []string{""}
-	for k := range b.repo.SubRepoMap {
-		paths = append(paths, k)
-	}
-	sort.Strings(paths)
-	b.subRepoIndices = make(map[string]uint32, len(paths))
-	for i, p := range paths {
-		b.subRepoIndices[p] = uint32(i)
-	}
-}
-
-const notIndexedMarker = "NOT-INDEXED: "
-
-// Add a file which only occurs in certain branches.
-func (b *IndexBuilder) Add(doc Document) error {
-	hasher := crc64.New(crc64.MakeTable(crc64.ISO))
-
-	if idx := bytes.IndexByte(doc.Content, 0); idx >= 0 {
-		doc.SkipReason = fmt.Sprintf("binary content at byte offset %d", idx)
-		doc.Language = "binary"
-	}
-
-	if doc.SkipReason != "" {
-		doc.Content = []byte(notIndexedMarker + doc.SkipReason)
-		doc.Symbols = nil
-		if doc.Language == "" {
-			doc.Language = "skipped"
-		}
-	}
-
-	sort.Sort(docSectionSlice(doc.Symbols))
-	var last DocumentSection
-	for i, s := range doc.Symbols {
-		if i > 0 {
-			if last.End > s.Start {
-				return fmt.Errorf("sections overlap")
-			}
-		}
-		last = s
-	}
-	if last.End > uint32(len(doc.Content)) {
-		return fmt.Errorf("section goes past end of content")
-	}
-
-	if doc.SubRepositoryPath != "" {
-		rel, err := filepath.Rel(doc.SubRepositoryPath, doc.Name)
-		if err != nil || rel == doc.Name {
-			return fmt.Errorf("path %q must start subrepo path %q", doc.Name, doc.SubRepositoryPath)
-		}
-	}
-	docStr, runeSecs, err := b.contentPostings.newSearchableString(doc.Content, doc.Symbols)
-	if err != nil {
-		return err
-	}
-	nameStr, _, err := b.namePostings.newSearchableString([]byte(doc.Name), nil)
-	if err != nil {
-		return err
-	}
-
-	subRepoIdx, ok := b.subRepoIndices[doc.SubRepositoryPath]
-	if !ok {
-		return fmt.Errorf("unknown subrepo path %q", doc.SubRepositoryPath)
-	}
-
-	var mask uint64
-	for _, br := range doc.Branches {
-		m := b.branchMask(br)
-		if m == 0 {
-			return fmt.Errorf("no branch found for %s", br)
-		}
-		mask |= m
-	}
-
-	b.subRepos = append(b.subRepos, subRepoIdx)
-
-	hasher.Write(doc.Content)
-
-	b.contentStrings = append(b.contentStrings, docStr)
-	b.runeDocSections = append(b.runeDocSections, runeSecs...)
-
-	b.nameStrings = append(b.nameStrings, nameStr)
-	b.docSections = append(b.docSections, doc.Symbols)
-	b.branchMasks = append(b.branchMasks, mask)
-	b.checksums = append(b.checksums, hasher.Sum(nil)...)
-
-	langCode, ok := b.languageMap[doc.Language]
-	if !ok {
-		if len(b.languageMap) >= 255 {
-			return fmt.Errorf("too many languages")
-		}
-		langCode = byte(len(b.languageMap))
-		b.languageMap[doc.Language] = langCode
-	}
-	b.languages = append(b.languages, langCode)
-
-	return nil
-}
-
-func (b *IndexBuilder) branchMask(br string) uint64 {
-	for i, b := range b.repo.Branches {
-		if b.Name == br {
-			return uint64(1) << uint(i)
-		}
-	}
-	return 0
-}
diff --git a/indexdata.go b/indexdata.go
deleted file mode 100644
index 4262116..0000000
--- a/indexdata.go
+++ /dev/null
@@ -1,275 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"fmt"
-	"hash/crc64"
-	"unicode/utf8"
-
-	"github.com/google/zoekt/query"
-)
-
-// indexData holds the pattern-independent data that we have to have
-// in memory to search. Most of the memory is taken up by the ngram =>
-// offset index.
-type indexData struct {
-	file IndexFile
-
-	ngrams map[ngram]simpleSection
-
-	newlinesStart uint32
-	newlinesIndex []uint32
-
-	docSectionsStart uint32
-	docSectionsIndex []uint32
-
-	runeDocSections []DocumentSection
-
-	// rune offset=>byte offset mapping, relative to the start of the content corpus
-	runeOffsets []uint32
-
-	// offsets of file contents; includes end of last file
-	boundariesStart uint32
-	boundaries      []uint32
-
-	// rune offsets for the file content boundaries
-	fileEndRunes []uint32
-
-	fileNameContent []byte
-	fileNameIndex   []uint32
-	fileNameNgrams  map[ngram][]uint32
-
-	// rune offset=>byte offset mapping, relative to the start of the filename corpus
-	fileNameRuneOffsets []uint32
-
-	// rune offsets for the file name boundaries
-	fileNameEndRunes []uint32
-
-	fileBranchMasks []uint64
-
-	// mask (power of 2) => name
-	branchNames map[uint]string
-
-	// name => mask (power of 2)
-	branchIDs map[string]uint
-
-	metaData     IndexMetadata
-	repoMetaData Repository
-
-	subRepos     []uint32
-	subRepoPaths []string
-
-	// Checksums for all the files, at 8-byte intervals
-	checksums []byte
-
-	// languages for all the files.
-	languages []byte
-
-	// inverse of LanguageMap in metaData
-	languageMap map[byte]string
-
-	repoListEntry RepoListEntry
-}
-
-func (d *indexData) getChecksum(idx uint32) []byte {
-	start := crc64.Size * idx
-	return d.checksums[start : start+crc64.Size]
-}
-
-func (d *indexData) calculateStats() {
-	var last uint32
-	if len(d.boundaries) > 0 {
-		last += d.boundaries[len(d.boundaries)-1]
-	}
-
-	lastFN := last
-	if len(d.fileNameIndex) > 0 {
-		lastFN = d.fileNameIndex[len(d.fileNameIndex)-1]
-	}
-
-	stats := RepoStats{
-		IndexBytes:   int64(d.memoryUse()),
-		ContentBytes: int64(int(last) + int(lastFN)),
-		Documents:    len(d.newlinesIndex) - 1,
-		Shards:       1,
-	}
-	d.repoListEntry = RepoListEntry{
-		Repository:    d.repoMetaData,
-		IndexMetadata: d.metaData,
-		Stats:         stats,
-	}
-}
-
-func (d *indexData) String() string {
-	return fmt.Sprintf("shard(%s)", d.file.Name())
-}
-
-func (d *indexData) memoryUse() int {
-	sz := 0
-	for _, a := range [][]uint32{
-		d.newlinesIndex, d.docSectionsIndex,
-		d.boundaries, d.fileNameIndex,
-		d.runeOffsets, d.fileNameRuneOffsets,
-		d.fileEndRunes, d.fileNameEndRunes,
-	} {
-		sz += 4 * len(a)
-	}
-	sz += 8 * len(d.runeDocSections)
-	sz += 8 * len(d.fileBranchMasks)
-	sz += 12 * len(d.ngrams)
-	for _, v := range d.fileNameNgrams {
-		sz += 4*len(v) + 4
-	}
-	return sz
-}
-
-const maxUInt32 = 0xffffffff
-
-func firstMinarg(xs []uint32) uint32 {
-	m := uint32(maxUInt32)
-	j := len(xs)
-	for i, x := range xs {
-		if x < m {
-			m = x
-			j = i
-		}
-	}
-	return uint32(j)
-}
-
-func lastMinarg(xs []uint32) uint32 {
-	m := uint32(maxUInt32)
-	j := len(xs)
-	for i, x := range xs {
-		if x <= m {
-			m = x
-			j = i
-		}
-	}
-	return uint32(j)
-}
-
-func (data *indexData) ngramFrequency(ng ngram, filename bool) uint32 {
-	if filename {
-		return uint32(len(data.fileNameNgrams[ng]))
-	}
-
-	return data.ngrams[ng].sz
-}
-
-type ngramIterationResults struct {
-	matchIterator
-
-	caseSensitive bool
-	fileName      bool
-	substrBytes   []byte
-	substrLowered []byte
-}
-
-func (r *ngramIterationResults) String() string {
-	return fmt.Sprintf("wrapper(%v)", r.matchIterator)
-}
-
-func (r *ngramIterationResults) candidates() []*candidateMatch {
-	cs := r.matchIterator.candidates()
-	for _, c := range cs {
-		c.caseSensitive = r.caseSensitive
-		c.fileName = r.fileName
-		c.substrBytes = r.substrBytes
-		c.substrLowered = r.substrLowered
-	}
-	return cs
-}
-
-func (d *indexData) iterateNgrams(query *query.Substring) (*ngramIterationResults, error) {
-	str := query.Pattern
-
-	// Find the 2 least common ngrams from the string.
-	ngramOffs := splitNGrams([]byte(query.Pattern))
-	frequencies := make([]uint32, 0, len(ngramOffs))
-	for _, o := range ngramOffs {
-		var freq uint32
-		if query.CaseSensitive {
-			freq = d.ngramFrequency(o.ngram, query.FileName)
-		} else {
-			for _, v := range generateCaseNgrams(o.ngram) {
-				freq += d.ngramFrequency(v, query.FileName)
-			}
-		}
-
-		if freq == 0 {
-			return &ngramIterationResults{
-				matchIterator: &noMatchTree{
-					Why: "freq=0",
-				},
-			}, nil
-		}
-
-		frequencies = append(frequencies, freq)
-	}
-	firstI := firstMinarg(frequencies)
-	frequencies[firstI] = maxUInt32
-	lastI := lastMinarg(frequencies)
-	if firstI > lastI {
-		lastI, firstI = firstI, lastI
-	}
-
-	firstNG := ngramOffs[firstI].ngram
-	lastNG := ngramOffs[lastI].ngram
-	iter := &ngramDocIterator{
-		leftPad:  firstI,
-		rightPad: uint32(utf8.RuneCountInString(str)) - firstI,
-	}
-	if query.FileName {
-		iter.ends = d.fileNameEndRunes
-	} else {
-		iter.ends = d.fileEndRunes
-	}
-
-	if firstI != lastI {
-		i, err := d.newDistanceTrigramIter(firstNG, lastNG, lastI-firstI, query.CaseSensitive, query.FileName)
-		if err != nil {
-			return nil, err
-		}
-
-		iter.iter = i
-	} else {
-		hitIter, err := d.trigramHitIterator(lastNG, query.CaseSensitive, query.FileName)
-		if err != nil {
-			return nil, err
-		}
-		iter.iter = hitIter
-	}
-
-	patBytes := []byte(query.Pattern)
-	lowerPatBytes := toLower(patBytes)
-
-	return &ngramIterationResults{
-		matchIterator: iter,
-		caseSensitive: query.CaseSensitive,
-		fileName:      query.FileName,
-		substrBytes:   patBytes,
-		substrLowered: lowerPatBytes,
-	}, nil
-}
-
-func (d *indexData) fileName(i uint32) []byte {
-	return d.fileNameContent[d.fileNameIndex[i]:d.fileNameIndex[i+1]]
-}
-
-func (s *indexData) Close() {
-	s.file.Close()
-}
diff --git a/indexfile_other.go b/indexfile_other.go
deleted file mode 100644
index 1c03fe8..0000000
--- a/indexfile_other.go
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build !linux,!darwin
-
-package zoekt
-
-import (
-	"fmt"
-	"os"
-)
-
-// NewIndexFile returns a new index file. The index file takes
-// ownership of the passed in file, and may close it.
-func NewIndexFile(f *os.File) (IndexFile, error) {
-	return &indexFileFromOS{f}, nil
-}
-
-type indexFileFromOS struct {
-	f *os.File
-}
-
-func (f *indexFileFromOS) Read(off, sz uint32) ([]byte, error) {
-	r := make([]byte, sz)
-	_, err := f.f.ReadAt(r, int64(off))
-	return r, err
-}
-
-func (f indexFileFromOS) Size() (uint32, error) {
-	fi, err := f.f.Stat()
-	if err != nil {
-		return 0, err
-	}
-
-	sz := fi.Size()
-
-	if sz >= maxUInt32 {
-		return 0, fmt.Errorf("overflow")
-	}
-
-	return uint32(sz), nil
-}
-
-func (f indexFileFromOS) Close() {
-	f.f.Close()
-}
-
-func (f indexFileFromOS) Name() string {
-	return f.f.Name()
-}
diff --git a/indexfile_unix.go b/indexfile_unix.go
deleted file mode 100644
index d7e9402..0000000
--- a/indexfile_unix.go
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build linux darwin
-
-package zoekt
-
-import (
-	"fmt"
-	"os"
-	"syscall"
-)
-
-type mmapedIndexFile struct {
-	name string
-	size uint32
-	data []byte
-}
-
-func (f *mmapedIndexFile) Read(off, sz uint32) ([]byte, error) {
-	if off+sz > uint32(len(f.data)) {
-		return nil, fmt.Errorf("out of bounds: %d, len %d", off+sz, len(f.data))
-	}
-	return f.data[off : off+sz], nil
-}
-
-func (f *mmapedIndexFile) Name() string {
-	return f.name
-}
-
-func (f *mmapedIndexFile) Size() (uint32, error) {
-	return f.size, nil
-}
-
-func (f *mmapedIndexFile) Close() {
-	syscall.Munmap(f.data)
-}
-
-// NewIndexFile returns a new index file. The index file takes
-// ownership of the passed in file, and may close it.
-func NewIndexFile(f *os.File) (IndexFile, error) {
-	defer f.Close()
-
-	fi, err := f.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	sz := fi.Size()
-	if sz >= maxUInt32 {
-		return nil, fmt.Errorf("file %s too large: %d", f.Name(), sz)
-	}
-	r := &mmapedIndexFile{
-		name: f.Name(),
-		size: uint32(sz),
-	}
-
-	rounded := (r.size + 4095) &^ 4095
-	r.data, err = syscall.Mmap(int(f.Fd()), 0, int(rounded), syscall.PROT_READ, syscall.MAP_SHARED)
-	if err != nil {
-		return nil, err
-	}
-
-	return r, err
-}
diff --git a/matchiter.go b/matchiter.go
deleted file mode 100644
index c76e1b8..0000000
--- a/matchiter.go
+++ /dev/null
@@ -1,280 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"bytes"
-	"fmt"
-	"sort"
-	"unicode/utf8"
-
-	"github.com/google/zoekt/query"
-)
-
-// candidateMatch is a candidate match for a substring.
-type candidateMatch struct {
-	caseSensitive bool
-	fileName      bool
-
-	substrBytes   []byte
-	substrLowered []byte
-
-	file uint32
-
-	// Offsets are relative to the start of the filename or file contents.
-	runeOffset  uint32
-	byteOffset  uint32
-	byteMatchSz uint32
-}
-
-// Matches content against the substring, and populates byteMatchSz on success
-func (m *candidateMatch) matchContent(content []byte) bool {
-	if m.caseSensitive {
-		comp := bytes.Equal(m.substrBytes, content[m.byteOffset:m.byteOffset+uint32(len(m.substrBytes))])
-
-		m.byteMatchSz = uint32(len(m.substrBytes))
-		return comp
-	} else {
-		// It is tempting to try a simple ASCII based
-		// comparison if possible, but we need more
-		// information. Simple ASCII chars have unicode upper
-		// case variants (the ASCII 'k' has the Kelvin symbol
-		// as upper case variant). We can only degrade to
-		// ASCII if we are sure that both the corpus and the
-		// query is ASCII only
-		sz, ok := caseFoldingEqualsRunes(m.substrLowered, content[m.byteOffset:])
-		m.byteMatchSz = uint32(sz)
-		return ok
-	}
-}
-
-// line returns the line holding the match. If the match starts with
-// the newline ending line M, we return M.  The line is characterized
-// by its linenumber (base-1, byte index of line start, byte index of
-// line end).  The line end is the index of a newline, or the filesize
-// (if matching the last line of the file.)
-func (m *candidateMatch) line(newlines []uint32, fileSize uint32) (lineNum, lineStart, lineEnd int) {
-	idx := sort.Search(len(newlines), func(n int) bool {
-		return newlines[n] >= m.byteOffset
-	})
-
-	end := int(fileSize)
-	if idx < len(newlines) {
-		end = int(newlines[idx])
-	}
-
-	start := 0
-	if idx > 0 {
-		start = int(newlines[idx-1] + 1)
-	}
-
-	return idx + 1, start, end
-}
-
-// matchIterator is a docIterator that produces candidateMatches for a given document
-type matchIterator interface {
-	docIterator
-
-	candidates() []*candidateMatch
-	updateStats(*Stats)
-}
-
-// noMatchTree is both matchIterator and matchTree that matches nothing.
-type noMatchTree struct {
-	Why string
-}
-
-func (t *noMatchTree) String() string {
-	return fmt.Sprintf("not(%q)", t.Why)
-}
-
-func (t *noMatchTree) candidates() []*candidateMatch {
-	return nil
-}
-
-func (t *noMatchTree) nextDoc() uint32 {
-	return maxUInt32
-}
-
-func (t *noMatchTree) prepare(uint32) {}
-
-func (t *noMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	return false, true
-}
-
-func (t *noMatchTree) updateStats(*Stats) {}
-
-func (m *candidateMatch) String() string {
-	return fmt.Sprintf("%d:%d", m.file, m.runeOffset)
-}
-
-type ngramDocIterator struct {
-	leftPad  uint32
-	rightPad uint32
-
-	iter hitIterator
-	ends []uint32
-
-	// mutable
-	fileIdx    uint32
-	matchCount int
-}
-
-// nextFileIndex returns the smallest index j of ends such that
-// ends[j] > offset, assuming ends[f] <= offset.
-func nextFileIndex(offset, f uint32, ends []uint32) uint32 {
-	d := uint32(1)
-	for f < uint32(len(ends)) && ends[f] <= offset {
-		if f+d < uint32(len(ends)) && ends[f+d] <= offset {
-			f += d
-			d *= 2
-		} else if d > 1 {
-			d = d/4 + 1
-		} else {
-			f++
-		}
-	}
-	return f
-}
-
-func (i *ngramDocIterator) nextDoc() uint32 {
-	i.fileIdx = nextFileIndex(i.iter.first(), i.fileIdx, i.ends)
-	if i.fileIdx >= uint32(len(i.ends)) {
-		return maxUInt32
-	}
-	return i.fileIdx
-}
-
-func (i *ngramDocIterator) String() string {
-	return fmt.Sprintf("ngram(L=%d,R=%d,%v)", i.leftPad, i.rightPad, i.iter)
-}
-
-func (i *ngramDocIterator) prepare(nextDoc uint32) {
-	var start uint32
-	if nextDoc > 0 {
-		start = i.ends[nextDoc-1]
-	}
-	if start > 0 {
-		i.iter.next(start + i.leftPad - 1)
-	}
-	i.fileIdx = nextDoc
-}
-
-func (i *ngramDocIterator) updateStats(s *Stats) {
-	i.iter.updateStats(s)
-	s.NgramMatches += i.matchCount
-}
-
-func (i *ngramDocIterator) candidates() []*candidateMatch {
-	if i.fileIdx >= uint32(len(i.ends)) {
-		return nil
-	}
-
-	var fileStart uint32
-	if i.fileIdx > 0 {
-		fileStart = i.ends[i.fileIdx-1]
-	}
-	fileEnd := i.ends[i.fileIdx]
-
-	var candidates []*candidateMatch
-	for {
-		p1 := i.iter.first()
-		if p1 == maxUInt32 || p1 >= i.ends[i.fileIdx] {
-			break
-		}
-		i.iter.next(p1)
-
-		if p1 < i.leftPad+fileStart || p1+i.rightPad > fileEnd {
-			continue
-		}
-
-		candidates = append(candidates, &candidateMatch{
-			file:       uint32(i.fileIdx),
-			runeOffset: p1 - fileStart - i.leftPad,
-		})
-	}
-	i.matchCount += len(candidates)
-	return candidates
-}
-
-type trimBySectionMatchIter struct {
-	matchIterator
-
-	patternSize  uint32
-	fileEndRunes []uint32
-
-	// mutable
-	doc      uint32
-	sections []DocumentSection
-}
-
-func (i *trimBySectionMatchIter) String() string {
-	return fmt.Sprintf("trimSection(sz=%d, %v)", i.patternSize, i.matchIterator)
-}
-
-func (d *indexData) newTrimByDocSectionIter(q *query.Substring, iter matchIterator) *trimBySectionMatchIter {
-	return &trimBySectionMatchIter{
-		matchIterator: iter,
-		patternSize:   uint32(utf8.RuneCountInString(q.Pattern)),
-		fileEndRunes:  d.fileEndRunes,
-		sections:      d.runeDocSections,
-	}
-}
-
-func (i *trimBySectionMatchIter) prepare(doc uint32) {
-	i.matchIterator.prepare(doc)
-	i.doc = doc
-
-	var fileStart uint32
-	if doc > 0 {
-		fileStart = i.fileEndRunes[doc-1]
-	}
-
-	for len(i.sections) > 0 && i.sections[0].Start < fileStart {
-		i.sections = i.sections[1:]
-	}
-}
-
-func (i *trimBySectionMatchIter) candidates() []*candidateMatch {
-	var fileStart uint32
-	if i.doc > 0 {
-		fileStart = i.fileEndRunes[i.doc-1]
-	}
-
-	ms := i.matchIterator.candidates()
-	trimmed := ms[:0]
-	for len(i.sections) > 0 && len(ms) > 0 {
-		start := fileStart + ms[0].runeOffset
-		end := start + i.patternSize
-		if start >= i.sections[0].End {
-			i.sections = i.sections[1:]
-			continue
-		}
-
-		if start < i.sections[0].Start {
-			ms = ms[1:]
-			continue
-		}
-
-		// here we have: sec.Start <= start < sec.End
-		if end <= i.sections[0].End {
-			// complete match falls inside section.
-			trimmed = append(trimmed, ms[0])
-		}
-
-		ms = ms[1:]
-	}
-	return trimmed
-}
diff --git a/matchtree.go b/matchtree.go
deleted file mode 100644
index 66d9372..0000000
--- a/matchtree.go
+++ /dev/null
@@ -1,743 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"fmt"
-	"log"
-	"regexp"
-	"strings"
-	"unicode/utf8"
-
-	"github.com/google/zoekt/query"
-)
-
-// A docIterator iterates over documents in order.
-type docIterator interface {
-	// provide the next document where we can may find something
-	// interesting.
-	nextDoc() uint32
-
-	// clears any per-document state of the docIterator, and
-	// prepares for evaluating the given doc. The argument is
-	// strictly increasing over time.
-	prepare(nextDoc uint32)
-}
-
-const (
-	costConst   = 0
-	costMemory  = 1
-	costContent = 2
-	costRegexp  = 3
-)
-
-const (
-	costMin = costConst
-	costMax = costRegexp
-)
-
-// An expression tree coupled with matches. The matchtree has two
-// functions:
-//
-// * it implements boolean combinations (and, or, not)
-//
-// * it implements shortcuts, where we skip documents (for example: if
-// there are no trigram matches, we can be sure there are no substring
-// matches). The matchtree iterates over the documents as they are
-// ordered in the shard.
-//
-// The general process for a given (shard, query) is
-//
-// - construct matchTree for the query
-//
-// - find all different leaf matchTrees (substring, regexp, etc.)
-//
-// in a loop:
-//
-//   - find next doc to process using nextDoc
-//
-//   - evaluate atoms (leaf expressions that match text)
-//
-//   - evaluate the tree using matches(), storing the result in map.
-//
-//   - if the complete tree returns (matches() == true) for the document,
-//     collect all text matches by looking at leaf matchTrees
-//
-type matchTree interface {
-	docIterator
-
-	// returns whether this matches, and if we are sure.
-	matches(cp *contentProvider, cost int, known map[matchTree]bool) (match bool, sure bool)
-}
-
-// docMatchTree iterates over documents for which predicate(docID) returns true.
-type docMatchTree struct {
-	// the number of documents in a shard.
-	numDocs uint32
-
-	predicate func(docID uint32) bool
-
-	// provides additional information about the reason why the docMatchTree was
-	// created.
-	reason string
-
-	// mutable
-	firstDone bool
-	docID     uint32
-}
-
-type bruteForceMatchTree struct {
-	// mutable
-	firstDone bool
-	docID     uint32
-}
-
-type andLineMatchTree struct {
-	andMatchTree
-}
-
-type andMatchTree struct {
-	children []matchTree
-}
-
-type orMatchTree struct {
-	children []matchTree
-}
-
-type notMatchTree struct {
-	child matchTree
-}
-
-// Don't visit this subtree for collecting matches.
-type noVisitMatchTree struct {
-	matchTree
-}
-
-type regexpMatchTree struct {
-	regexp *regexp.Regexp
-
-	fileName bool
-
-	// mutable
-	reEvaluated bool
-	found       []*candidateMatch
-
-	// nextDoc, prepare.
-	bruteForceMatchTree
-}
-
-type substrMatchTree struct {
-	matchIterator
-
-	query         *query.Substring
-	caseSensitive bool
-	fileName      bool
-
-	// mutable
-	current       []*candidateMatch
-	contEvaluated bool
-}
-
-type branchQueryMatchTree struct {
-	fileMasks []uint64
-	mask      uint64
-
-	// mutable
-	firstDone bool
-	docID     uint32
-}
-
-// all prepare methods
-
-func (t *bruteForceMatchTree) prepare(doc uint32) {
-	t.docID = doc
-	t.firstDone = true
-}
-
-func (t *docMatchTree) prepare(doc uint32) {
-	t.docID = doc
-	t.firstDone = true
-}
-
-func (t *andMatchTree) prepare(doc uint32) {
-	for _, c := range t.children {
-		c.prepare(doc)
-	}
-}
-
-func (t *regexpMatchTree) prepare(doc uint32) {
-	t.found = t.found[:0]
-	t.reEvaluated = false
-	t.bruteForceMatchTree.prepare(doc)
-}
-
-func (t *orMatchTree) prepare(doc uint32) {
-	for _, c := range t.children {
-		c.prepare(doc)
-	}
-}
-
-func (t *notMatchTree) prepare(doc uint32) {
-	t.child.prepare(doc)
-}
-
-func (t *substrMatchTree) prepare(nextDoc uint32) {
-	t.matchIterator.prepare(nextDoc)
-	t.current = t.matchIterator.candidates()
-	t.contEvaluated = false
-}
-
-func (t *branchQueryMatchTree) prepare(doc uint32) {
-	t.firstDone = true
-	t.docID = doc
-}
-
-// nextDoc
-
-func (t *docMatchTree) nextDoc() uint32 {
-	var start uint32
-	if t.firstDone {
-		start = t.docID + 1
-	}
-	for i := start; i < t.numDocs; i++ {
-		if t.predicate(i) {
-			return i
-		}
-	}
-	return maxUInt32
-}
-
-func (t *bruteForceMatchTree) nextDoc() uint32 {
-	if !t.firstDone {
-		return 0
-	}
-	return t.docID + 1
-}
-
-func (t *andMatchTree) nextDoc() uint32 {
-	var max uint32
-	for _, c := range t.children {
-		m := c.nextDoc()
-		if m > max {
-			max = m
-		}
-	}
-	return max
-}
-
-func (t *orMatchTree) nextDoc() uint32 {
-	min := uint32(maxUInt32)
-	for _, c := range t.children {
-		m := c.nextDoc()
-		if m < min {
-			min = m
-		}
-	}
-	return min
-}
-
-func (t *notMatchTree) nextDoc() uint32 {
-	return 0
-}
-
-func (t *branchQueryMatchTree) nextDoc() uint32 {
-	var start uint32
-	if t.firstDone {
-		start = t.docID + 1
-	}
-
-	for i := start; i < uint32(len(t.fileMasks)); i++ {
-		if (t.mask & t.fileMasks[i]) != 0 {
-			return i
-		}
-	}
-	return maxUInt32
-}
-
-// all String methods
-
-func (t *bruteForceMatchTree) String() string {
-	return "all"
-}
-
-func (t *docMatchTree) String() string {
-	return fmt.Sprintf("doc(%s)", t.reason)
-}
-
-func (t *andMatchTree) String() string {
-	return fmt.Sprintf("and%v", t.children)
-}
-
-func (t *regexpMatchTree) String() string {
-	return fmt.Sprintf("re(%s)", t.regexp)
-}
-
-func (t *orMatchTree) String() string {
-	return fmt.Sprintf("or%v", t.children)
-}
-
-func (t *notMatchTree) String() string {
-	return fmt.Sprintf("not(%v)", t.child)
-}
-
-func (t *substrMatchTree) String() string {
-	f := ""
-	if t.fileName {
-		f = "f"
-	}
-
-	return fmt.Sprintf("%ssubstr(%q, %v, %v)", f, t.query.Pattern, t.current, t.matchIterator)
-}
-
-func (t *branchQueryMatchTree) String() string {
-	return fmt.Sprintf("branch(%x)", t.mask)
-}
-
-// visitMatches visits all atoms in matchTree. Note: This visits
-// noVisitMatchTree. For collecting matches use visitMatches.
-func visitMatchTree(t matchTree, f func(matchTree)) {
-	switch s := t.(type) {
-	case *andMatchTree:
-		for _, ch := range s.children {
-			visitMatchTree(ch, f)
-		}
-	case *orMatchTree:
-		for _, ch := range s.children {
-			visitMatchTree(ch, f)
-		}
-	case *andLineMatchTree:
-		visitMatchTree(&s.andMatchTree, f)
-	case *noVisitMatchTree:
-		visitMatchTree(s.matchTree, f)
-	case *notMatchTree:
-		visitMatchTree(s.child, f)
-	default:
-		f(t)
-	}
-}
-
-// visitMatches visits all atoms which can contribute matches. Note: This
-// skips noVisitMatchTree.
-func visitMatches(t matchTree, known map[matchTree]bool, f func(matchTree)) {
-	switch s := t.(type) {
-	case *andMatchTree:
-		for _, ch := range s.children {
-			if known[ch] {
-				visitMatches(ch, known, f)
-			}
-		}
-	case *andLineMatchTree:
-		visitMatches(&s.andMatchTree, known, f)
-	case *orMatchTree:
-		for _, ch := range s.children {
-			if known[ch] {
-				visitMatches(ch, known, f)
-			}
-		}
-	case *notMatchTree:
-	case *noVisitMatchTree:
-		// don't collect into negative trees.
-	default:
-		f(s)
-	}
-}
-
-// all matches() methods.
-
-func (t *docMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	return t.predicate(cp.idx), true
-}
-
-func (t *bruteForceMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	return true, true
-}
-
-// andLineMatchTree is a performance optimization of andMatchTree. For content
-// searches we don't want to run the regex engine if there is no line that
-// contains matches from all terms.
-func (t *andLineMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	matches, sure := t.andMatchTree.matches(cp, cost, known)
-	if !(sure && matches) {
-		return matches, sure
-	}
-
-	// find child with fewest candidates
-	min := maxUInt32
-	fewestChildren := 0
-	for ix, child := range t.children {
-		v, ok := child.(*substrMatchTree)
-		// make sure we are running a content search and that all candidates are a
-		// substrMatchTree
-		if !ok || v.fileName {
-			return matches, sure
-		}
-		if len(v.current) < min {
-			min = len(v.current)
-			fewestChildren = ix
-		}
-	}
-
-	type lineRange struct {
-		start int
-		end   int
-	}
-	lines := make([]lineRange, 0, len(t.children[fewestChildren].(*substrMatchTree).current))
-	prev := -1
-	for _, candidate := range t.children[fewestChildren].(*substrMatchTree).current {
-		line, byteStart, byteEnd := candidate.line(cp.newlines(), cp.fileSize)
-		if line == prev {
-			continue
-		}
-		prev = line
-		lines = append(lines, lineRange{byteStart, byteEnd})
-	}
-
-	// children keeps track of the children's candidates we have already seen.
-	children := make([][]*candidateMatch, 0, len(t.children)-1)
-	for j, child := range t.children {
-		if j == fewestChildren {
-			continue
-		}
-		children = append(children, child.(*substrMatchTree).current)
-	}
-
-nextLine:
-	for i := 0; i < len(lines); i++ {
-		hits := 1
-	nextChild:
-		for j := range children {
-		nextCandidate:
-			for len(children[j]) > 0 {
-				candidate := children[j][0]
-				bo := int(cp.findOffset(false, candidate.runeOffset))
-				if bo < lines[i].start {
-					children[j] = children[j][1:]
-					continue nextCandidate
-				}
-				if bo <= lines[i].end {
-					hits++
-					continue nextChild
-				}
-				// move the `lines` iterator forward until bo <= line.end
-				for i < len(lines) && bo > lines[i].end {
-					i++
-				}
-				i--
-				continue nextLine
-			}
-		}
-		// return early once we found any line that contains matches from all children
-		if hits == len(t.children) {
-			return matches, true
-		}
-	}
-	return false, true
-}
-
-func (t *andMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	sure := true
-
-	for _, ch := range t.children {
-		v, ok := evalMatchTree(cp, cost, known, ch)
-		if ok && !v {
-			return false, true
-		}
-		if !ok {
-			sure = false
-		}
-	}
-
-	return true, sure
-}
-
-func (t *orMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	matches := false
-	sure := true
-	for _, ch := range t.children {
-		v, ok := evalMatchTree(cp, cost, known, ch)
-		if ok {
-			// we could short-circuit, but we want to use
-			// the other possibilities as a ranking
-			// signal.
-			matches = matches || v
-		} else {
-			sure = false
-		}
-	}
-	return matches, sure
-}
-
-func (t *branchQueryMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	return t.fileMasks[t.docID]&t.mask != 0, true
-}
-
-func (t *regexpMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	if t.reEvaluated {
-		return len(t.found) > 0, true
-	}
-
-	if cost < costRegexp {
-		return false, false
-	}
-
-	cp.stats.RegexpsConsidered++
-	idxs := t.regexp.FindAllIndex(cp.data(t.fileName), -1)
-	found := t.found[:0]
-	for _, idx := range idxs {
-		cm := &candidateMatch{
-			byteOffset:  uint32(idx[0]),
-			byteMatchSz: uint32(idx[1] - idx[0]),
-			fileName:    t.fileName,
-		}
-
-		found = append(found, cm)
-	}
-	t.found = found
-	t.reEvaluated = true
-
-	return len(t.found) > 0, true
-}
-
-// breakMatchesOnNewlines returns matches resulting from breaking each element
-// of cms on newlines within text.
-func breakMatchesOnNewlines(cms []*candidateMatch, text []byte) []*candidateMatch {
-	var lineCMs []*candidateMatch
-	for _, cm := range cms {
-		lineCMs = append(lineCMs, breakOnNewlines(cm, text)...)
-	}
-	return lineCMs
-}
-
-// breakOnNewlines returns matches resulting from breaking cm on newlines
-// within text.
-func breakOnNewlines(cm *candidateMatch, text []byte) []*candidateMatch {
-	var cms []*candidateMatch
-	addMe := &candidateMatch{}
-	*addMe = *cm
-	for i := uint32(cm.byteOffset); i < cm.byteOffset+cm.byteMatchSz; i++ {
-		if text[i] == '\n' {
-			addMe.byteMatchSz = i - addMe.byteOffset
-			if addMe.byteMatchSz != 0 {
-				cms = append(cms, addMe)
-			}
-
-			addMe = &candidateMatch{}
-			*addMe = *cm
-			addMe.byteOffset = i + 1
-		}
-	}
-	addMe.byteMatchSz = cm.byteOffset + cm.byteMatchSz - addMe.byteOffset
-	if addMe.byteMatchSz != 0 {
-		cms = append(cms, addMe)
-	}
-	return cms
-}
-
-func evalMatchTree(cp *contentProvider, cost int, known map[matchTree]bool, mt matchTree) (bool, bool) {
-	if v, ok := known[mt]; ok {
-		return v, true
-	}
-
-	v, ok := mt.matches(cp, cost, known)
-	if ok {
-		known[mt] = v
-	}
-
-	return v, ok
-}
-
-func (t *notMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	v, ok := evalMatchTree(cp, cost, known, t.child)
-	return !v, ok
-}
-
-func (t *substrMatchTree) matches(cp *contentProvider, cost int, known map[matchTree]bool) (bool, bool) {
-	if t.contEvaluated {
-		return len(t.current) > 0, true
-	}
-
-	if len(t.current) == 0 {
-		return false, true
-	}
-
-	if t.fileName && cost < costMemory {
-		return false, false
-	}
-
-	if !t.fileName && cost < costContent {
-		return false, false
-	}
-
-	pruned := t.current[:0]
-	for _, m := range t.current {
-		if m.byteOffset == 0 && m.runeOffset > 0 {
-			m.byteOffset = cp.findOffset(m.fileName, m.runeOffset)
-		}
-		if m.matchContent(cp.data(m.fileName)) {
-			pruned = append(pruned, m)
-		}
-	}
-	t.current = pruned
-	t.contEvaluated = true
-
-	return len(t.current) > 0, true
-}
-
-func (d *indexData) newMatchTree(q query.Q) (matchTree, error) {
-	if q == nil {
-		return nil, fmt.Errorf("got nil (sub)query")
-	}
-	switch s := q.(type) {
-	case *query.Regexp:
-		// RegexpToMatchTreeRecursive tries to distill a matchTree that matches a
-		// superset of the regexp. If the returned matchTree is equivalent to the
-		// original regexp, it returns true. An equivalent matchTree has the same
-		// behaviour as the original regexp and can be used instead.
-		//
-		subMT, isEq, _, err := d.regexpToMatchTreeRecursive(s.Regexp, ngramSize, s.FileName, s.CaseSensitive)
-		if err != nil {
-			return nil, err
-		}
-		// if the query can be used in place of the regexp
-		// return the subtree
-		if isEq {
-			return subMT, nil
-		}
-
-		prefix := ""
-		if !s.CaseSensitive {
-			prefix = "(?i)"
-		}
-
-		tr := &regexpMatchTree{
-			regexp:   regexp.MustCompile(prefix + s.Regexp.String()),
-			fileName: s.FileName,
-		}
-
-		return &andMatchTree{
-			children: []matchTree{
-				tr, &noVisitMatchTree{subMT},
-			},
-		}, nil
-	case *query.And:
-		var r []matchTree
-		for _, ch := range s.Children {
-			ct, err := d.newMatchTree(ch)
-			if err != nil {
-				return nil, err
-			}
-			r = append(r, ct)
-		}
-		return &andMatchTree{r}, nil
-	case *query.Or:
-		var r []matchTree
-		for _, ch := range s.Children {
-			ct, err := d.newMatchTree(ch)
-			if err != nil {
-				return nil, err
-			}
-			r = append(r, ct)
-		}
-		return &orMatchTree{r}, nil
-	case *query.Not:
-		ct, err := d.newMatchTree(s.Child)
-		return &notMatchTree{
-			child: ct,
-		}, err
-
-	case *query.Substring:
-		return d.newSubstringMatchTree(s)
-
-	case *query.Branch:
-		mask := uint64(0)
-		if s.Pattern == "HEAD" {
-			mask = 1
-		} else {
-			for nm, m := range d.branchIDs {
-				if strings.Contains(nm, s.Pattern) {
-					mask |= uint64(m)
-				}
-			}
-		}
-		return &branchQueryMatchTree{
-			mask:      mask,
-			fileMasks: d.fileBranchMasks,
-		}, nil
-	case *query.Const:
-		if s.Value {
-			return &bruteForceMatchTree{}, nil
-		} else {
-			return &noMatchTree{"const"}, nil
-		}
-	case *query.Language:
-		code, ok := d.metaData.LanguageMap[s.Language]
-		if !ok {
-			return &noMatchTree{"lang"}, nil
-		}
-		return &docMatchTree{
-			reason:  "language",
-			numDocs: uint32(len(d.languages)),
-			predicate: func(docID uint32) bool {
-				return d.languages[docID] == code
-			},
-		}, nil
-
-	case *query.Symbol:
-		mt, err := d.newSubstringMatchTree(s.Atom)
-		if err != nil {
-			return nil, err
-		}
-
-		if _, ok := mt.(*regexpMatchTree); ok {
-			return nil, fmt.Errorf("regexps and short queries not implemented for symbol search")
-		}
-		subMT, ok := mt.(*substrMatchTree)
-		if !ok {
-			return nil, fmt.Errorf("found %T inside query.Symbol", mt)
-		}
-
-		subMT.matchIterator = d.newTrimByDocSectionIter(s.Atom, subMT.matchIterator)
-		return subMT, nil
-	}
-	log.Panicf("type %T", q)
-	return nil, nil
-}
-
-func (d *indexData) newSubstringMatchTree(s *query.Substring) (matchTree, error) {
-	st := &substrMatchTree{
-		query:         s,
-		caseSensitive: s.CaseSensitive,
-		fileName:      s.FileName,
-	}
-
-	if utf8.RuneCountInString(s.Pattern) < ngramSize {
-		prefix := ""
-		if !s.CaseSensitive {
-			prefix = "(?i)"
-		}
-		t := &regexpMatchTree{
-			regexp:   regexp.MustCompile(prefix + regexp.QuoteMeta(s.Pattern)),
-			fileName: s.FileName,
-		}
-		return t, nil
-	}
-
-	result, err := d.iterateNgrams(s)
-	if err != nil {
-		return nil, err
-	}
-	st.matchIterator = result
-	return st, nil
-}
diff --git a/matchtree_test.go b/matchtree_test.go
deleted file mode 100644
index 07aa4a5..0000000
--- a/matchtree_test.go
+++ /dev/null
@@ -1,191 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/google/zoekt/query"
-)
-
-func Test_breakOnNewlines(t *testing.T) {
-	type args struct {
-		cm   *candidateMatch
-		text []byte
-	}
-	tests := []struct {
-		name string
-		args args
-		want []*candidateMatch
-	}{
-		{
-			name: "trivial case",
-			args: args{
-				cm: &candidateMatch{
-					byteOffset:  0,
-					byteMatchSz: 0,
-				},
-				text: nil,
-			},
-			want: nil,
-		},
-		{
-			name: "no newlines",
-			args: args{
-				cm: &candidateMatch{
-					byteOffset:  0,
-					byteMatchSz: 1,
-				},
-				text: []byte("a"),
-			},
-			want: []*candidateMatch{
-				{
-					byteOffset:  0,
-					byteMatchSz: 1,
-				},
-			},
-		},
-		{
-			name: "newline at start",
-			args: args{
-				cm: &candidateMatch{
-					byteOffset:  0,
-					byteMatchSz: 2,
-				},
-				text: []byte("\na"),
-			},
-			want: []*candidateMatch{
-				{
-					byteOffset:  1,
-					byteMatchSz: 1,
-				},
-			},
-		},
-		{
-			name: "newline at end",
-			args: args{
-				cm: &candidateMatch{
-					byteOffset:  0,
-					byteMatchSz: 2,
-				},
-				text: []byte("a\n"),
-			},
-			want: []*candidateMatch{
-				{
-					byteOffset:  0,
-					byteMatchSz: 1,
-				},
-			},
-		},
-		{
-			name: "newline in middle",
-			args: args{
-				cm: &candidateMatch{
-					byteOffset:  0,
-					byteMatchSz: 3,
-				},
-				text: []byte("a\nb"),
-			},
-			want: []*candidateMatch{
-				{
-					byteOffset:  0,
-					byteMatchSz: 1,
-				},
-				{
-					byteOffset:  2,
-					byteMatchSz: 1,
-				},
-			},
-		},
-		{
-			name: "two newlines",
-			args: args{
-				cm: &candidateMatch{
-					byteOffset:  0,
-					byteMatchSz: 5,
-				},
-				text: []byte("a\nb\nc"),
-			},
-			want: []*candidateMatch{
-				{
-					byteOffset:  0,
-					byteMatchSz: 1,
-				},
-				{
-					byteOffset:  2,
-					byteMatchSz: 1,
-				},
-				{
-					byteOffset:  4,
-					byteMatchSz: 1,
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := breakOnNewlines(tt.args.cm, tt.args.text); !reflect.DeepEqual(got, tt.want) {
-				type PrintableCm struct {
-					byteOffset  uint32
-					byteMatchSz uint32
-				}
-				var got2, want2 []PrintableCm
-				for _, g := range got {
-					got2 = append(got2, PrintableCm{byteOffset: g.byteOffset, byteMatchSz: g.byteMatchSz})
-				}
-				for _, w := range tt.want {
-					want2 = append(want2, PrintableCm{byteOffset: w.byteOffset, byteMatchSz: w.byteMatchSz})
-				}
-				t.Errorf("breakMatchOnNewlines() = %+v, want %+v", got2, want2)
-			}
-		})
-	}
-}
-
-func TestEquivalentQuerySkipRegexpTree(t *testing.T) {
-	tests := []struct {
-		query string
-		skip  bool
-	}{
-		{query: "^foo", skip: false},
-		{query: "foo", skip: true},
-		{query: "thread|needle|haystack", skip: true},
-		{query: "contain(er|ing)", skip: false},
-		{query: "thread (needle|haystack)", skip: true},
-		{query: "thread (needle|)", skip: false},
-	}
-
-	for _, tt := range tests {
-		q, err := query.Parse(tt.query)
-		if err != nil {
-			t.Errorf("Error parsing query: %s", "sym:"+tt.query)
-			continue
-		}
-
-		d := &indexData{}
-		mt, err := d.newMatchTree(q)
-		if err != nil {
-			t.Errorf("Error creating match tree from query: %s", q)
-			continue
-		}
-
-		visitMatchTree(mt, func(m matchTree) {
-			if _, ok := m.(*regexpMatchTree); ok && tt.skip {
-				t.Errorf("Expected regexpMatchTree to be skipped for query: %s", q)
-			}
-		})
-	}
-}
diff --git a/query/bits.go b/query/bits.go
deleted file mode 100644
index 8b199d2..0000000
--- a/query/bits.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package query
-
-func toLower(in []byte) []byte {
-	out := make([]byte, len(in))
-	for i, c := range in {
-		if c >= 'A' && c <= 'Z' {
-			c = c - 'A' + 'a'
-		}
-		out[i] = c
-	}
-	return out
-}
diff --git a/query/parse.go b/query/parse.go
deleted file mode 100644
index ecbebf7..0000000
--- a/query/parse.go
+++ /dev/null
@@ -1,474 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package query
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"regexp/syntax"
-)
-
-var _ = log.Printf
-
-type SuggestQueryError struct {
-	Message    string
-	Suggestion string
-}
-
-func (e *SuggestQueryError) Error() string {
-	return fmt.Sprintf("%s. Suggestion: %s", e.Message, e.Suggestion)
-}
-
-// parseStringLiteral parses a string literal, consumes the starting
-// quote too.
-func parseStringLiteral(in []byte) (lit []byte, n int, err error) {
-	left := in[1:]
-	found := false
-
-loop:
-	for len(left) > 0 {
-		c := left[0]
-		left = left[1:]
-		switch c {
-		case '"':
-			found = true
-			break loop
-		case '\\':
-			// TODO - other escape sequences.
-			if len(left) == 0 {
-				return nil, 0, fmt.Errorf("query: missing char after \\")
-			}
-			c = left[0]
-			left = left[1:]
-
-			lit = append(lit, c)
-		default:
-			lit = append(lit, c)
-		}
-	}
-	if !found {
-		return nil, 0, fmt.Errorf("query: unterminated quoted string")
-	}
-	return lit, len(in) - len(left), nil
-}
-
-// orOperator is a placeholder intermediate so we can represent [A,
-// or, B] before we convert it to Or{A, B}
-type orOperator struct{}
-
-func (o *orOperator) String() string {
-	return "orOp"
-}
-
-func isSpace(c byte) bool {
-	return c == ' ' || c == '\t'
-}
-
-// Parse parses a string into a query.
-func Parse(qStr string) (Q, error) {
-	b := []byte(qStr)
-
-	qs, _, err := parseExprList(b)
-	if err != nil {
-		return nil, err
-	}
-
-	q, err := parseOperators(qs)
-	if err != nil {
-		return nil, err
-	}
-
-	return Simplify(q), nil
-}
-
-// parseExpr parses a single expression, returning the result, and the
-// number of bytes consumed.
-func parseExpr(in []byte) (Q, int, error) {
-	b := in[:]
-	var expr Q
-	for len(b) > 0 && isSpace(b[0]) {
-		b = b[1:]
-	}
-
-	tok, err := nextToken(b)
-	if err != nil {
-		return nil, 0, err
-	}
-	if tok == nil {
-		return nil, 0, nil
-	}
-	b = b[len(tok.Input):]
-
-	text := string(tok.Text)
-	switch tok.Type {
-	case tokCase:
-		switch text {
-		case "yes":
-		case "no":
-		case "auto":
-		default:
-			return nil, 0, fmt.Errorf("query: unknown case argument %q, want {yes,no,auto}", text)
-		}
-		expr = &caseQ{text}
-	case tokRepo:
-		expr = &Repo{Pattern: text}
-	case tokBranch:
-		expr = &Branch{Pattern: text}
-	case tokText, tokRegex:
-		q, err := regexpQuery(text, false, false)
-		if err != nil {
-			return nil, 0, err
-		}
-		expr = q
-	case tokFile:
-		q, err := regexpQuery(text, false, true)
-		if err != nil {
-			return nil, 0, err
-		}
-		expr = q
-
-	case tokContent:
-		q, err := regexpQuery(text, true, false)
-		if err != nil {
-			return nil, 0, err
-		}
-		expr = q
-	case tokLang:
-		expr = &Language{Language: text}
-
-	case tokSym:
-		if text == "" {
-			return nil, 0, fmt.Errorf("the sym: atom must have an argument")
-		}
-		expr = &Symbol{&Substring{Pattern: text}}
-
-	case tokParenClose:
-		// Caller must consume paren.
-		expr = nil
-
-	case tokParenOpen:
-		qs, n, err := parseExprList(b)
-		b = b[n:]
-		if err != nil {
-			return nil, 0, err
-		}
-
-		pTok, err := nextToken(b)
-		if err != nil {
-			return nil, 0, err
-		}
-		if pTok == nil || pTok.Type != tokParenClose {
-			return nil, 0, fmt.Errorf("query: missing close paren, got token %v", pTok)
-		}
-
-		b = b[len(pTok.Input):]
-		expr, err = parseOperators(qs)
-		if err != nil {
-			return nil, 0, err
-		}
-	case tokNegate:
-		subQ, n, err := parseExpr(b)
-		if err != nil {
-			return nil, 0, err
-		}
-		if subQ == nil {
-			return nil, 0, fmt.Errorf("query: '-' operator needs an argument")
-		}
-		b = b[n:]
-		expr = &Not{subQ}
-
-	}
-
-	return expr, len(in) - len(b), nil
-}
-
-// regexpQuery parses an atom into either a regular expression, or a
-// simple substring atom.
-func regexpQuery(text string, content, file bool) (Q, error) {
-	var expr Q
-
-	r, err := syntax.Parse(text, syntax.ClassNL|syntax.PerlX|syntax.UnicodeGroups)
-	if err != nil {
-		return nil, err
-	}
-
-	if r.Op == syntax.OpLiteral {
-		expr = &Substring{
-			Pattern:  string(r.Rune),
-			FileName: file,
-			Content:  content,
-		}
-	} else {
-		expr = &Regexp{
-			Regexp:   r,
-			FileName: file,
-			Content:  content,
-		}
-	}
-
-	return expr, nil
-}
-
-// parseOperators interprets the orOperator in a list of queries.
-func parseOperators(in []Q) (Q, error) {
-	top := &Or{}
-	cur := &And{}
-
-	seenOr := false
-	for _, q := range in {
-		if _, ok := q.(*orOperator); ok {
-			seenOr = true
-			if len(cur.Children) == 0 {
-				return nil, fmt.Errorf("query: OR operator should have operand")
-			}
-			top.Children = append(top.Children, cur)
-			cur = &And{}
-		} else {
-			cur.Children = append(cur.Children, q)
-		}
-	}
-
-	if seenOr && len(cur.Children) == 0 {
-		return nil, fmt.Errorf("query: OR operator should have operand")
-	}
-	top.Children = append(top.Children, cur)
-	return top, nil
-}
-
-// parseExprList parses a list of query expressions. It is the
-// workhorse of the Parse function.
-func parseExprList(in []byte) ([]Q, int, error) {
-	b := in[:]
-	var qs []Q
-	for len(b) > 0 {
-		for len(b) > 0 && isSpace(b[0]) {
-			b = b[1:]
-		}
-		tok, _ := nextToken(b)
-		if tok != nil && tok.Type == tokParenClose {
-			break
-		} else if tok != nil && tok.Type == tokOr {
-			qs = append(qs, &orOperator{})
-			b = b[len(tok.Input):]
-			continue
-		}
-
-		q, n, err := parseExpr(b)
-		if err != nil {
-			return nil, 0, err
-		}
-
-		if q == nil {
-			// eof or a ')'
-			break
-		}
-		qs = append(qs, q)
-		b = b[n:]
-	}
-
-	setCase := "auto"
-	newQS := qs[:0]
-	for _, q := range qs {
-		if sc, ok := q.(*caseQ); ok {
-			setCase = sc.Flavor
-		} else {
-			newQS = append(newQS, q)
-		}
-	}
-	qs = mapQueryList(newQS, func(q Q) Q {
-		if sc, ok := q.(setCaser); ok {
-			sc.setCase(setCase)
-		}
-		return q
-	})
-	return qs, len(in) - len(b), nil
-}
-
-type token struct {
-	Type int
-	// The value of the token
-	Text []byte
-
-	// The input that we consumed to form the token.
-	Input []byte
-}
-
-func (t *token) String() string {
-	return fmt.Sprintf("%s:%q", tokNames[t.Type], t.Text)
-}
-
-// token types.
-const (
-	tokText       = 0
-	tokFile       = 1
-	tokRepo       = 2
-	tokCase       = 3
-	tokBranch     = 4
-	tokParenOpen  = 5
-	tokParenClose = 6
-	tokError      = 7
-	tokNegate     = 8
-	tokRegex      = 9
-	tokOr         = 10
-	tokContent    = 11
-	tokLang       = 12
-	tokSym        = 13
-)
-
-var tokNames = map[int]string{
-	tokBranch:     "Branch",
-	tokCase:       "Case",
-	tokError:      "Error",
-	tokFile:       "File",
-	tokNegate:     "Negate",
-	tokOr:         "Or",
-	tokParenClose: "ParenClose",
-	tokParenOpen:  "ParenOpen",
-	tokRegex:      "Regex",
-	tokRepo:       "Repo",
-	tokText:       "Text",
-	tokLang:       "Language",
-	tokSym:        "Symbol",
-}
-
-var prefixes = map[string]int{
-	"b:":       tokBranch,
-	"branch:":  tokBranch,
-	"c:":       tokContent,
-	"case:":    tokCase,
-	"content:": tokContent,
-	"f:":       tokFile,
-	"file:":    tokFile,
-	"r:":       tokRepo,
-	"regex:":   tokRegex,
-	"repo:":    tokRepo,
-	"lang:":    tokLang,
-	"sym:":     tokSym,
-}
-
-var reservedWords = map[string]int{
-	"or": tokOr,
-}
-
-func (t *token) setType() {
-	// After we consumed the input, we have to interpret some of the text,
-	// eg. to distinguish between ")" the text and ) the query grouping
-	// parenthesis.
-	if len(t.Text) == 1 && t.Text[0] == '(' {
-		t.Type = tokParenOpen
-	}
-	if len(t.Text) == 1 && t.Text[0] == ')' {
-		t.Type = tokParenClose
-	}
-
-	for w, typ := range reservedWords {
-		if string(t.Text) == w && string(t.Input) == w {
-			t.Type = typ
-			break
-		}
-	}
-
-	for pref, typ := range prefixes {
-		if !bytes.HasPrefix(t.Input, []byte(pref)) {
-			continue
-		}
-
-		t.Text = t.Text[len(pref):]
-		t.Type = typ
-		break
-	}
-}
-
-// nextToken returns the next token from the given input.
-func nextToken(in []byte) (*token, error) {
-	left := in[:]
-	parenCount := 0
-	var cur token
-	if len(left) == 0 {
-		return nil, nil
-	}
-
-	if left[0] == '-' {
-		return &token{
-			Type:  tokNegate,
-			Text:  []byte{'-'},
-			Input: in[:1],
-		}, nil
-	}
-
-	foundSpace := false
-
-loop:
-	for len(left) > 0 {
-		c := left[0]
-		switch c {
-		case '(':
-			parenCount++
-			cur.Text = append(cur.Text, c)
-			left = left[1:]
-		case ')':
-			if parenCount == 0 {
-				if len(cur.Text) == 0 {
-					cur.Text = []byte{')'}
-					left = left[1:]
-				}
-				break loop
-			}
-
-			cur.Text = append(cur.Text, c)
-			left = left[1:]
-			parenCount--
-
-		case '"':
-			t, n, err := parseStringLiteral(left)
-			if err != nil {
-				return nil, err
-			}
-			cur.Text = append(cur.Text, t...)
-			left = left[n:]
-		case '\\':
-			left = left[1:]
-			if len(left) == 0 {
-				return nil, fmt.Errorf("query: lone \\ at end")
-			}
-			c := left[0]
-			cur.Text = append(cur.Text, '\\', c)
-			left = left[1:]
-
-		case ' ', '\n', '\t':
-			if parenCount > 0 {
-				foundSpace = true
-			}
-			break loop
-		default:
-			cur.Text = append(cur.Text, c)
-			left = left[1:]
-		}
-	}
-
-	if len(cur.Text) == 0 {
-		return nil, nil
-	}
-
-	if foundSpace && cur.Text[0] == '(' {
-		cur.Text = cur.Text[:1]
-		cur.Input = in[:1]
-	} else {
-		cur.Input = in[:len(in)-len(left)]
-	}
-	cur.setType()
-	return &cur, nil
-}
diff --git a/query/parse_test.go b/query/parse_test.go
deleted file mode 100644
index 8acd719..0000000
--- a/query/parse_test.go
+++ /dev/null
@@ -1,164 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package query
-
-import (
-	"log"
-	"reflect"
-	"regexp/syntax"
-	"testing"
-)
-
-func mustParseRE(s string) *syntax.Regexp {
-	r, err := syntax.Parse(s, syntax.ClassNL|syntax.PerlX|syntax.UnicodeGroups)
-	if err != nil {
-		log.Panicf("parsing %q: %v", s, err)
-	}
-	return r
-}
-
-func TestParseQuery(t *testing.T) {
-	type testcase struct {
-		in   string
-		want Q
-	}
-
-	for _, c := range []testcase{
-		{`\bword\b`, &Regexp{Regexp: mustParseRE(`\bword\b`)}},
-		{"fi\"le:bla\"", &Substring{Pattern: "file:bla"}},
-		{"abc or def", NewOr(&Substring{Pattern: "abc"}, &Substring{Pattern: "def"})},
-		{"(abc or def)", NewOr(&Substring{Pattern: "abc"}, &Substring{Pattern: "def"})},
-		{"(ppp qqq or rrr sss)", NewOr(
-			NewAnd(&Substring{Pattern: "ppp"}, &Substring{Pattern: "qqq"}),
-			NewAnd(&Substring{Pattern: "rrr"}, &Substring{Pattern: "sss"}))},
-		{"((x) ora b(z(d)))", NewAnd(
-			&Regexp{Regexp: mustParseRE("(x)")},
-			&Substring{Pattern: "ora"},
-			&Regexp{Regexp: mustParseRE("b(z(d))")})},
-		{"( )", &Const{Value: true}},
-		{"(abc)(de)", &Regexp{Regexp: mustParseRE("(abc)(de)")}},
-		{"sub-pixel", &Substring{Pattern: "sub-pixel"}},
-		{"abc", &Substring{Pattern: "abc"}},
-		{"ABC", &Substring{Pattern: "ABC", CaseSensitive: true}},
-		{"\"abc bcd\"", &Substring{Pattern: "abc bcd"}},
-		{"abc bcd", NewAnd(
-			&Substring{Pattern: "abc"},
-			&Substring{Pattern: "bcd"})},
-		{"f:fs", &Substring{Pattern: "fs", FileName: true}},
-		{"fs", &Substring{Pattern: "fs"}},
-		{"-abc", &Not{&Substring{Pattern: "abc"}}},
-		{"abccase:yes", &Substring{Pattern: "abccase:yes"}},
-		{"file:abc", &Substring{Pattern: "abc", FileName: true}},
-		{"branch:pqr", &Branch{Pattern: "pqr"}},
-		{"((x) )", &Regexp{Regexp: mustParseRE("(x)")}},
-		{"file:helpers\\.go byte", NewAnd(
-			&Substring{Pattern: "helpers.go", FileName: true},
-			&Substring{Pattern: "byte"})},
-		{"(abc def)", NewAnd(
-			&Substring{Pattern: "abc"},
-			&Substring{Pattern: "def"})},
-		{"(abc def", nil},
-		{"regex:abc[p-q]", &Regexp{Regexp: mustParseRE("abc[p-q]")}},
-		{"aBc[p-q]", &Regexp{Regexp: mustParseRE("aBc[p-q]"), CaseSensitive: true}},
-		{"aBc[p-q] case:auto", &Regexp{Regexp: mustParseRE("aBc[p-q]"), CaseSensitive: true}},
-		{"repo:go", &Repo{"go"}},
-
-		{"file:\"\"", &Const{true}},
-		{"abc.*def", &Regexp{Regexp: mustParseRE("abc.*def")}},
-		{"abc\\.\\*def", &Substring{Pattern: "abc.*def"}},
-		{"(abc)", &Regexp{Regexp: mustParseRE("(abc)")}},
-
-		{"c:abc", &Substring{Pattern: "abc", Content: true}},
-		{"content:abc", &Substring{Pattern: "abc", Content: true}},
-
-		{"lang:c++", &Language{"c++"}},
-		{"sym:pqr", &Symbol{&Substring{Pattern: "pqr"}}},
-		{"sym:Pqr", &Symbol{&Substring{Pattern: "Pqr", CaseSensitive: true}}},
-
-		// case
-		{"abc case:yes", &Substring{Pattern: "abc", CaseSensitive: true}},
-		{"abc case:auto", &Substring{Pattern: "abc", CaseSensitive: false}},
-		{"ABC case:auto", &Substring{Pattern: "ABC", CaseSensitive: true}},
-		{"ABC case:\"auto\"", &Substring{Pattern: "ABC", CaseSensitive: true}},
-		{"abc -f:def case:yes", NewAnd(
-			&Substring{Pattern: "abc", CaseSensitive: true},
-			&Not{Child: &Substring{Pattern: "def", FileName: true, CaseSensitive: true}},
-		)},
-
-		// errors.
-		{"--", nil},
-		{"\"abc", nil},
-		{"\"a\\", nil},
-		{"case:foo", nil},
-
-		{"sym:", nil},
-		{"abc or", nil},
-		{"or abc", nil},
-		{"def or or abc", nil},
-
-		{"", &Const{Value: true}},
-	} {
-		got, err := Parse(c.in)
-		if (c.want == nil) != (err != nil) {
-			t.Errorf("Parse(%q): error %v, want %v", c.in, err, c.want)
-		} else if got != nil {
-			if !reflect.DeepEqual(got, c.want) {
-				t.Errorf("Parse(%s): got %v want %v", c.in, got, c.want)
-			}
-		}
-	}
-}
-
-func TestTokenize(t *testing.T) {
-	type testcase struct {
-		in   string
-		typ  int
-		text string
-	}
-
-	cases := []testcase{
-		{"file:bla", tokFile, "bla"},
-		{"file:bla ", tokFile, "bla"},
-		{"f:bla ", tokFile, "bla"},
-		{"(abc def) ", tokParenOpen, "("},
-		{"(abcdef)", tokText, "(abcdef)"},
-		{"(abc)(de)", tokText, "(abc)(de)"},
-		{"(ab(c)def) ", tokText, "(ab(c)def)"},
-		{"(ab\\ def) ", tokText, "(ab\\ def)"},
-		{") ", tokParenClose, ")"},
-		{"a(bc))", tokText, "a(bc)"},
-		{"abc) ", tokText, "abc"},
-		{"file:\"bla\"", tokFile, "bla"},
-		{"\"file:bla\"", tokText, "file:bla"},
-		{"\\", tokError, ""},
-		{"o\"r\" bla", tokText, "or"},
-		{"or bla", tokOr, "or"},
-		{"ar bla", tokText, "ar"},
-	}
-	for _, c := range cases {
-		tok, err := nextToken([]byte(c.in))
-		if err != nil {
-			tok = &token{Type: tokError}
-		}
-		if tok.Type != c.typ {
-			t.Errorf("%s: got type %d, want %d", c.in, tok.Type, c.typ)
-			continue
-		}
-
-		if string(tok.Text) != c.text {
-			t.Errorf("%s: got text %q, want %q", c.in, tok.Text, c.text)
-		}
-	}
-}
diff --git a/query/query.go b/query/query.go
deleted file mode 100644
index 31741a0..0000000
--- a/query/query.go
+++ /dev/null
@@ -1,394 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package query
-
-import (
-	"fmt"
-	"log"
-	"reflect"
-	"regexp/syntax"
-	"strings"
-)
-
-var _ = log.Println
-
-// Q is a representation for a possibly hierarchical search query.
-type Q interface {
-	String() string
-}
-
-// RegexpQuery is a query looking for regular expressions matches.
-type Regexp struct {
-	Regexp        *syntax.Regexp
-	FileName      bool
-	Content       bool
-	CaseSensitive bool
-}
-
-// Symbol finds a string that is a symbol.
-type Symbol struct {
-	Atom *Substring
-}
-
-func (s *Symbol) String() string {
-	return fmt.Sprintf("sym:%s", s.Atom)
-}
-
-func (q *Regexp) String() string {
-	pref := ""
-	if q.FileName {
-		pref = "file_"
-	}
-	if q.CaseSensitive {
-		pref = "case_" + pref
-	}
-	return fmt.Sprintf("%sregex:%q", pref, q.Regexp.String())
-}
-
-type caseQ struct {
-	Flavor string
-}
-
-func (c *caseQ) String() string {
-	return "case:" + c.Flavor
-}
-
-type Language struct {
-	Language string
-}
-
-func (l *Language) String() string {
-	return "lang:" + l.Language
-}
-
-type Const struct {
-	Value bool
-}
-
-func (q *Const) String() string {
-	if q.Value {
-		return "TRUE"
-	}
-	return "FALSE"
-}
-
-type Repo struct {
-	Pattern string
-}
-
-func (q *Repo) String() string {
-	return fmt.Sprintf("repo:%s", q.Pattern)
-}
-
-// Substring is the most basic query: a query for a substring.
-type Substring struct {
-	Pattern       string
-	CaseSensitive bool
-
-	// Match only filename
-	FileName bool
-
-	// Match only content
-	Content bool
-}
-
-func (q *Substring) String() string {
-	s := ""
-
-	t := ""
-	if q.FileName {
-		t = "file_"
-	} else if q.Content {
-		t = "content_"
-	}
-
-	s += fmt.Sprintf("%ssubstr:%q", t, q.Pattern)
-	if q.CaseSensitive {
-		s = "case_" + s
-	}
-	return s
-}
-
-type setCaser interface {
-	setCase(string)
-}
-
-func (q *Substring) setCase(k string) {
-	switch k {
-	case "yes":
-		q.CaseSensitive = true
-	case "no":
-		q.CaseSensitive = false
-	case "auto":
-		// TODO - unicode
-		q.CaseSensitive = (q.Pattern != string(toLower([]byte(q.Pattern))))
-	}
-}
-
-func (q *Symbol) setCase(k string) {
-	q.Atom.setCase(k)
-}
-
-func (q *Regexp) setCase(k string) {
-	switch k {
-	case "yes":
-		q.CaseSensitive = true
-	case "no":
-		q.CaseSensitive = false
-	case "auto":
-		q.CaseSensitive = (q.Regexp.String() != LowerRegexp(q.Regexp).String())
-	}
-}
-
-// Or is matched when any of its children is matched.
-type Or struct {
-	Children []Q
-}
-
-func (q *Or) String() string {
-	var sub []string
-	for _, ch := range q.Children {
-		sub = append(sub, ch.String())
-	}
-	return fmt.Sprintf("(or %s)", strings.Join(sub, " "))
-}
-
-// Not inverts the meaning of its child.
-type Not struct {
-	Child Q
-}
-
-func (q *Not) String() string {
-	return fmt.Sprintf("(not %s)", q.Child)
-}
-
-// And is matched when all its children are.
-type And struct {
-	Children []Q
-}
-
-func (q *And) String() string {
-	var sub []string
-	for _, ch := range q.Children {
-		sub = append(sub, ch.String())
-	}
-	return fmt.Sprintf("(and %s)", strings.Join(sub, " "))
-}
-
-// NewAnd is syntactic sugar for constructing And queries.
-func NewAnd(qs ...Q) Q {
-	return &And{Children: qs}
-}
-
-// NewOr is syntactic sugar for constructing Or queries.
-func NewOr(qs ...Q) Q {
-	return &Or{Children: qs}
-}
-
-// Branch limits search to a specific branch.
-type Branch struct {
-	Pattern string
-}
-
-func (q *Branch) String() string {
-	return fmt.Sprintf("branch:%q", q.Pattern)
-}
-
-func queryChildren(q Q) []Q {
-	switch s := q.(type) {
-	case *And:
-		return s.Children
-	case *Or:
-		return s.Children
-	}
-	return nil
-}
-
-func flattenAndOr(children []Q, typ Q) ([]Q, bool) {
-	var flat []Q
-	changed := false
-	for _, ch := range children {
-		ch, subChanged := flatten(ch)
-		changed = changed || subChanged
-		if reflect.TypeOf(ch) == reflect.TypeOf(typ) {
-			changed = true
-			subChildren := queryChildren(ch)
-			if subChildren != nil {
-				flat = append(flat, subChildren...)
-			}
-		} else {
-			flat = append(flat, ch)
-		}
-	}
-
-	return flat, changed
-}
-
-// (and (and x y) z) => (and x y z) , the same for "or"
-func flatten(q Q) (Q, bool) {
-	switch s := q.(type) {
-	case *And:
-		if len(s.Children) == 1 {
-			return s.Children[0], true
-		}
-		flatChildren, changed := flattenAndOr(s.Children, s)
-		return &And{flatChildren}, changed
-	case *Or:
-		if len(s.Children) == 1 {
-			return s.Children[0], true
-		}
-		flatChildren, changed := flattenAndOr(s.Children, s)
-		return &Or{flatChildren}, changed
-	case *Not:
-		child, changed := flatten(s.Child)
-		return &Not{child}, changed
-	default:
-		return q, false
-	}
-}
-
-func mapQueryList(qs []Q, f func(Q) Q) []Q {
-	neg := make([]Q, len(qs))
-	for i, sub := range qs {
-		neg[i] = Map(sub, f)
-	}
-	return neg
-}
-
-func invertConst(q Q) Q {
-	c, ok := q.(*Const)
-	if ok {
-		return &Const{!c.Value}
-	}
-	return q
-}
-
-func evalAndOrConstants(q Q, children []Q) Q {
-	_, isAnd := q.(*And)
-
-	children = mapQueryList(children, evalConstants)
-
-	newCH := children[:0]
-	for _, ch := range children {
-		c, ok := ch.(*Const)
-		if ok {
-			if c.Value == isAnd {
-				continue
-			} else {
-				return ch
-			}
-		}
-		newCH = append(newCH, ch)
-	}
-	if len(newCH) == 0 {
-		return &Const{isAnd}
-	}
-	if isAnd {
-		return &And{newCH}
-	}
-	return &Or{newCH}
-}
-
-func evalConstants(q Q) Q {
-	switch s := q.(type) {
-	case *And:
-		return evalAndOrConstants(q, s.Children)
-	case *Or:
-		return evalAndOrConstants(q, s.Children)
-	case *Not:
-		ch := evalConstants(s.Child)
-		if _, ok := ch.(*Const); ok {
-			return invertConst(ch)
-		}
-		return &Not{ch}
-	case *Substring:
-		if len(s.Pattern) == 0 {
-			return &Const{true}
-		}
-	case *Regexp:
-		if s.Regexp.Op == syntax.OpEmptyMatch {
-			return &Const{true}
-		}
-	case *Branch:
-		if s.Pattern == "" {
-			return &Const{true}
-		}
-	}
-	return q
-}
-
-func Simplify(q Q) Q {
-	q = evalConstants(q)
-	for {
-		var changed bool
-		q, changed = flatten(q)
-		if !changed {
-			break
-		}
-	}
-
-	return q
-}
-
-// Map runs f over the q.
-func Map(q Q, f func(q Q) Q) Q {
-	switch s := q.(type) {
-	case *And:
-		q = &And{Children: mapQueryList(s.Children, f)}
-	case *Or:
-		q = &Or{Children: mapQueryList(s.Children, f)}
-	case *Not:
-		q = &Not{Child: Map(s.Child, f)}
-	}
-	return f(q)
-}
-
-// Expand expands Substr queries into (OR file_substr content_substr)
-// queries, and the same for Regexp queries..
-func ExpandFileContent(q Q) Q {
-	switch s := q.(type) {
-	case *Substring:
-		if !s.FileName && !s.Content {
-			f := *s
-			f.FileName = true
-			c := *s
-			c.Content = true
-			return NewOr(&f, &c)
-		}
-	case *Regexp:
-		if !s.FileName && !s.Content {
-			f := *s
-			f.FileName = true
-			c := *s
-			c.Content = true
-			return NewOr(&f, &c)
-		}
-	}
-	return q
-}
-
-// VisitAtoms runs `v` on all atom queries within `q`.
-func VisitAtoms(q Q, v func(q Q)) {
-	Map(q, func(iQ Q) Q {
-		switch iQ.(type) {
-		case *And:
-		case *Or:
-		case *Not:
-		default:
-			v(iQ)
-		}
-		return iQ
-	})
-}
diff --git a/query/query_test.go b/query/query_test.go
deleted file mode 100644
index 0d85b29..0000000
--- a/query/query_test.go
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package query
-
-import (
-	"log"
-	"reflect"
-	"testing"
-)
-
-var _ = log.Println
-
-func TestQueryString(t *testing.T) {
-	q := &Or{[]Q{
-		&And{[]Q{
-			&Substring{Pattern: "hoi"},
-			&Not{&Substring{Pattern: "hai"}},
-		}},
-	}}
-	got := q.String()
-	want := `(or (and substr:"hoi" (not substr:"hai")))`
-
-	if got != want {
-		t.Errorf("got %s, want %s", got, want)
-	}
-}
-
-func TestSimplify(t *testing.T) {
-	type testcase struct {
-		in   Q
-		want Q
-	}
-
-	cases := []testcase{
-		{
-			in: NewOr(
-				NewOr(
-					NewAnd(&Substring{Pattern: "hoi"},
-						&Not{&Substring{Pattern: "hai"}}),
-					NewOr(
-						&Substring{Pattern: "zip"},
-						&Substring{Pattern: "zap"},
-					))),
-			want: NewOr(
-				NewAnd(
-					&Substring{Pattern: "hoi"},
-					&Not{&Substring{Pattern: "hai"}}),
-				&Substring{Pattern: "zip"},
-				&Substring{Pattern: "zap"}),
-		},
-		{in: &And{}, want: &Const{true}},
-		{in: &Or{}, want: &Const{false}},
-		{in: NewAnd(&Const{true}, &Const{false}), want: &Const{false}},
-		{in: NewOr(&Const{false}, &Const{true}), want: &Const{true}},
-		{in: &Not{&Const{true}}, want: &Const{false}},
-		{
-			in: NewAnd(
-				&Substring{Pattern: "byte"},
-				&Not{NewAnd(&Substring{Pattern: "byte"})}),
-			want: NewAnd(
-				&Substring{Pattern: "byte"},
-				&Not{&Substring{Pattern: "byte"}}),
-		},
-	}
-
-	for _, c := range cases {
-		got := Simplify(c.in)
-		if !reflect.DeepEqual(got, c.want) {
-			t.Errorf("got %s, want %s", got, c.want)
-		}
-	}
-}
-
-func TestMap(t *testing.T) {
-	in := NewAnd(&Substring{Pattern: "bla"}, &Not{&Repo{"foo"}})
-	out := NewAnd(&Substring{Pattern: "bla"}, &Not{&Const{false}})
-
-	f := func(q Q) Q {
-		if _, ok := q.(*Repo); ok {
-			return &Const{false}
-		}
-		return q
-	}
-	got := Map(in, f)
-	if !reflect.DeepEqual(got, out) {
-		t.Errorf("got %v, want %v", got, out)
-	}
-}
-
-func TestVisitAtoms(t *testing.T) {
-	in := NewAnd(&Substring{}, &Repo{}, &Not{&Const{}})
-	count := 0
-	VisitAtoms(in, func(q Q) {
-		count++
-	})
-	if count != 3 {
-		t.Errorf("got %d, want 3", count)
-	}
-}
diff --git a/query/regexp.go b/query/regexp.go
deleted file mode 100644
index 64c9def..0000000
--- a/query/regexp.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package query
-
-import (
-	"log"
-	"regexp/syntax"
-)
-
-var _ = log.Println
-
-func LowerRegexp(r *syntax.Regexp) *syntax.Regexp {
-	newRE := *r
-	switch r.Op {
-	case syntax.OpLiteral, syntax.OpCharClass:
-		newRE.Rune = make([]rune, len(r.Rune))
-		for i, c := range r.Rune {
-			if c >= 'A' && c <= 'Z' {
-				newRE.Rune[i] = c + 'a' - 'A'
-			} else {
-				newRE.Rune[i] = c
-			}
-		}
-	default:
-		newRE.Sub = make([]*syntax.Regexp, len(newRE.Sub))
-		for i, s := range r.Sub {
-			newRE.Sub[i] = LowerRegexp(s)
-		}
-	}
-
-	return &newRE
-}
diff --git a/query/regexp_test.go b/query/regexp_test.go
deleted file mode 100644
index a3a12a0..0000000
--- a/query/regexp_test.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package query
-
-import (
-	"regexp/syntax"
-	"strings"
-	"testing"
-)
-
-var opnames = map[syntax.Op]string{
-	syntax.OpNoMatch:        "OpNoMatch",
-	syntax.OpEmptyMatch:     "OpEmptyMatch",
-	syntax.OpLiteral:        "OpLiteral",
-	syntax.OpCharClass:      "OpCharClass",
-	syntax.OpAnyCharNotNL:   "OpAnyCharNotNL",
-	syntax.OpAnyChar:        "OpAnyChar",
-	syntax.OpBeginLine:      "OpBeginLine",
-	syntax.OpEndLine:        "OpEndLine",
-	syntax.OpBeginText:      "OpBeginText",
-	syntax.OpEndText:        "OpEndText",
-	syntax.OpWordBoundary:   "OpWordBoundary",
-	syntax.OpNoWordBoundary: "OpNoWordBoundary",
-	syntax.OpCapture:        "OpCapture",
-	syntax.OpStar:           "OpStar",
-	syntax.OpPlus:           "OpPlus",
-	syntax.OpQuest:          "OpQuest",
-	syntax.OpRepeat:         "OpRepeat",
-	syntax.OpConcat:         "OpConcat",
-	syntax.OpAlternate:      "OpAlternate",
-}
-
-func printRegexp(t *testing.T, r *syntax.Regexp, lvl int) {
-	t.Logf("%s%s ch: %d", strings.Repeat(" ", lvl), opnames[r.Op], len(r.Sub))
-	for _, s := range r.Sub {
-		printRegexp(t, s, lvl+1)
-	}
-}
-
-func TestLowerRegexp(t *testing.T) {
-	in := "[a-zA-Z]fooBAR"
-	re := mustParseRE(in)
-	in = re.String()
-	got := LowerRegexp(re)
-	want := "[a-za-z]foobar"
-	if got.String() != want {
-		printRegexp(t, re, 0)
-		printRegexp(t, got, 0)
-		t.Errorf("got %s, want %s", got, want)
-	}
-
-	if re.String() != in {
-		t.Errorf("got mutated original %s want %s", re.String(), in)
-	}
-}
diff --git a/read.go b/read.go
deleted file mode 100644
index 8351a23..0000000
--- a/read.go
+++ /dev/null
@@ -1,484 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"encoding/binary"
-	"encoding/json"
-	"fmt"
-	"log"
-	"sort"
-)
-
-// IndexFile is a file suitable for concurrent read access. For performance
-// reasons, it allows a mmap'd implementation.
-type IndexFile interface {
-	Read(off uint32, sz uint32) ([]byte, error)
-	Size() (uint32, error)
-	Close()
-	Name() string
-}
-
-// reader is a stateful file
-type reader struct {
-	r   IndexFile
-	off uint32
-}
-
-func (r *reader) seek(off uint32) {
-	r.off = off
-}
-
-func (r *reader) U32() (uint32, error) {
-	b, err := r.r.Read(r.off, 4)
-	r.off += 4
-	if err != nil {
-		return 0, err
-	}
-	return binary.BigEndian.Uint32(b), nil
-}
-
-func (r *reader) U64() (uint64, error) {
-	b, err := r.r.Read(r.off, 8)
-	r.off += 8
-	if err != nil {
-		return 0, err
-	}
-	return binary.BigEndian.Uint64(b), nil
-}
-
-func (r *reader) ReadByte() (byte, error) {
-	b, err := r.r.Read(r.off, 1)
-	r.off += 1
-	if err != nil {
-		return 0, err
-	}
-	return b[0], nil
-}
-
-func (r *reader) Varint() (uint64, error) {
-	v, err := binary.ReadUvarint(r)
-	if err != nil {
-		return 0, err
-	}
-	return v, nil
-}
-
-func (r *reader) Str() (string, error) {
-	slen, err := r.Varint()
-	if err != nil {
-		return "", err
-	}
-	b, err := r.r.Read(r.off, uint32(slen))
-	if err != nil {
-		return "", err
-	}
-	r.off += uint32(slen)
-	return string(b), nil
-}
-
-func (r *reader) readTOC(toc *indexTOC) error {
-	sz, err := r.r.Size()
-	if err != nil {
-		return err
-	}
-	r.off = sz - 8
-
-	var tocSection simpleSection
-	if err := tocSection.read(r); err != nil {
-		return err
-	}
-
-	r.seek(tocSection.off)
-
-	sectionCount, err := r.U32()
-	if err != nil {
-		return err
-	}
-
-	if sectionCount == 0 {
-		// tagged sections are indicated by a 0 sectionCount,
-		// and then a list of string-tagged type-indicated sections.
-		secs := toc.sectionsTagged()
-		for r.off < tocSection.off+tocSection.sz {
-			tag, err := r.Str()
-			if err != nil {
-				return err
-			}
-			kind, err := r.Varint()
-			if err != nil {
-				return err
-			}
-			sec := secs[tag]
-			if sec != nil && sec.kind() == sectionKind(kind) {
-				// happy path
-				if err := sec.read(r); err != nil {
-					return err
-				}
-				continue
-			}
-			// error case: skip over unknown section
-			if sec == nil {
-				log.Printf("file %s TOC has unknown section %q", r.r.Name(), tag)
-			} else {
-				return fmt.Errorf("file %s TOC section %q expects kind %d, got kind %d", r.r.Name(), tag,
-					kind, sec.kind())
-			}
-			if kind == 0 {
-				(&simpleSection{}).read(r)
-			} else if kind == 1 {
-				(&compoundSection{}).read(r)
-			}
-		}
-	} else {
-		// TODO: Remove this branch when ReaderMinFeatureVersion >= 10
-
-		secs := toc.sections()
-
-		if len(secs) != int(sectionCount) {
-			return fmt.Errorf("section count mismatch: got %d want %d", sectionCount, len(secs))
-		}
-
-		for _, s := range secs {
-			if err := s.read(r); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (r *indexData) readSectionBlob(sec simpleSection) ([]byte, error) {
-	return r.file.Read(sec.off, sec.sz)
-}
-
-func readSectionU32(f IndexFile, sec simpleSection) ([]uint32, error) {
-	if sec.sz%4 != 0 {
-		return nil, fmt.Errorf("barf: section size %% 4 != 0: sz %d ", sec.sz)
-	}
-	blob, err := f.Read(sec.off, sec.sz)
-	if err != nil {
-		return nil, err
-	}
-	arr := make([]uint32, 0, len(blob)/4)
-	for len(blob) > 0 {
-		arr = append(arr, binary.BigEndian.Uint32(blob))
-		blob = blob[4:]
-	}
-	return arr, nil
-}
-
-func readSectionU64(f IndexFile, sec simpleSection) ([]uint64, error) {
-	if sec.sz%8 != 0 {
-		return nil, fmt.Errorf("barf: section size %% 8 != 0: sz %d ", sec.sz)
-	}
-	blob, err := f.Read(sec.off, sec.sz)
-	if err != nil {
-		return nil, err
-	}
-	arr := make([]uint64, 0, len(blob)/8)
-	for len(blob) > 0 {
-		arr = append(arr, binary.BigEndian.Uint64(blob))
-		blob = blob[8:]
-	}
-	return arr, nil
-}
-
-func (r *reader) readJSON(data interface{}, sec *simpleSection) error {
-	blob, err := r.r.Read(sec.off, sec.sz)
-	if err != nil {
-		return err
-	}
-
-	return json.Unmarshal(blob, data)
-}
-
-func (r *reader) readIndexData(toc *indexTOC) (*indexData, error) {
-	d := indexData{
-		file:           r.r,
-		ngrams:         map[ngram]simpleSection{},
-		fileNameNgrams: map[ngram][]uint32{},
-		branchIDs:      map[string]uint{},
-		branchNames:    map[uint]string{},
-	}
-
-	blob, err := d.readSectionBlob(toc.metaData)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := json.Unmarshal(blob, &d.metaData); err != nil {
-		return nil, err
-	}
-
-	if d.metaData.IndexFormatVersion != IndexFormatVersion {
-		return nil, fmt.Errorf("file is v%d, want v%d", d.metaData.IndexFormatVersion, IndexFormatVersion)
-	}
-
-	if d.metaData.IndexFeatureVersion < ReadMinFeatureVersion {
-		return nil, fmt.Errorf("file is feature version %d, want feature version >= %d", d.metaData.IndexFeatureVersion, ReadMinFeatureVersion)
-	}
-
-	if d.metaData.IndexMinReaderVersion > FeatureVersion {
-		return nil, fmt.Errorf("file needs read feature version >= %d, have read feature version %d", d.metaData.IndexMinReaderVersion, FeatureVersion)
-	}
-
-	blob, err = d.readSectionBlob(toc.repoMetaData)
-	if err != nil {
-		return nil, err
-	}
-	if err := json.Unmarshal(blob, &d.repoMetaData); err != nil {
-		return nil, err
-	}
-
-	d.boundariesStart = toc.fileContents.data.off
-	d.boundaries = toc.fileContents.relativeIndex()
-	d.newlinesStart = toc.newlines.data.off
-	d.newlinesIndex = toc.newlines.relativeIndex()
-	d.docSectionsStart = toc.fileSections.data.off
-	d.docSectionsIndex = toc.fileSections.relativeIndex()
-
-	d.checksums, err = d.readSectionBlob(toc.contentChecksums)
-	if err != nil {
-		return nil, err
-	}
-
-	d.languages, err = d.readSectionBlob(toc.languages)
-	if err != nil {
-		return nil, err
-	}
-
-	d.ngrams, err = d.readNgrams(toc)
-	if err != nil {
-		return nil, err
-	}
-
-	d.fileBranchMasks, err = readSectionU64(d.file, toc.branchMasks)
-	if err != nil {
-		return nil, err
-	}
-
-	d.fileNameContent, err = d.readSectionBlob(toc.fileNames.data)
-	if err != nil {
-		return nil, err
-	}
-
-	d.fileNameIndex = toc.fileNames.relativeIndex()
-
-	d.fileNameNgrams, err = d.readFileNameNgrams(toc)
-	if err != nil {
-		return nil, err
-	}
-
-	for j, br := range d.repoMetaData.Branches {
-		id := uint(1) << uint(j)
-		d.branchIDs[br.Name] = id
-		d.branchNames[id] = br.Name
-	}
-
-	blob, err = d.readSectionBlob(toc.runeDocSections)
-	if err != nil {
-		return nil, err
-	}
-	d.runeDocSections = unmarshalDocSections(blob, nil)
-
-	for sect, dest := range map[simpleSection]*[]uint32{
-		toc.subRepos:        &d.subRepos,
-		toc.runeOffsets:     &d.runeOffsets,
-		toc.nameRuneOffsets: &d.fileNameRuneOffsets,
-		toc.nameEndRunes:    &d.fileNameEndRunes,
-		toc.fileEndRunes:    &d.fileEndRunes,
-	} {
-		if blob, err := d.readSectionBlob(sect); err != nil {
-			return nil, err
-		} else {
-			*dest = fromSizedDeltas(blob, nil)
-		}
-	}
-
-	keys := []string{""}
-	for k := range d.repoMetaData.SubRepoMap {
-		if k != "" { // we used to marshal "" in SubRepoMap. Prevent adding twice.
-			keys = append(keys, k)
-		}
-	}
-	sort.Strings(keys)
-	d.subRepoPaths = keys
-
-	d.languageMap = map[byte]string{}
-	for k, v := range d.metaData.LanguageMap {
-		d.languageMap[v] = k
-	}
-
-	if err := d.verify(); err != nil {
-		return nil, err
-	}
-
-	d.calculateStats()
-	return &d, nil
-}
-
-const ngramEncoding = 8
-
-func (d *indexData) readNgrams(toc *indexTOC) (map[ngram]simpleSection, error) {
-	textContent, err := d.readSectionBlob(toc.ngramText)
-	if err != nil {
-		return nil, err
-	}
-	postingsIndex := toc.postings.relativeIndex()
-
-	ngrams := make(map[ngram]simpleSection, len(textContent)/ngramEncoding)
-	for i := 0; i < len(textContent); i += ngramEncoding {
-		j := i / ngramEncoding
-		ng := ngram(binary.BigEndian.Uint64(textContent[i : i+ngramEncoding]))
-		ngrams[ng] = simpleSection{
-			toc.postings.data.off + postingsIndex[j],
-			postingsIndex[j+1] - postingsIndex[j],
-		}
-	}
-
-	return ngrams, nil
-}
-
-func (d *indexData) readFileNameNgrams(toc *indexTOC) (map[ngram][]uint32, error) {
-	nameNgramText, err := d.readSectionBlob(toc.nameNgramText)
-	if err != nil {
-		return nil, err
-	}
-
-	fileNamePostingsData, err := d.readSectionBlob(toc.namePostings.data)
-	if err != nil {
-		return nil, err
-	}
-
-	fileNamePostingsIndex := toc.namePostings.relativeIndex()
-
-	fileNameNgrams := make(map[ngram][]uint32, len(nameNgramText)/ngramEncoding)
-	for i := 0; i < len(nameNgramText); i += ngramEncoding {
-		j := i / ngramEncoding
-		off := fileNamePostingsIndex[j]
-		end := fileNamePostingsIndex[j+1]
-		ng := ngram(binary.BigEndian.Uint64(nameNgramText[i : i+ngramEncoding]))
-		fileNameNgrams[ng] = fromDeltas(fileNamePostingsData[off:end], nil)
-	}
-
-	return fileNameNgrams, nil
-}
-
-func (d *indexData) verify() error {
-	// This is not an exhaustive check: the postings can easily
-	// generate OOB acccesses, and are expensive to check, but this lets us rule out
-	// other sources of OOB access.
-	n := len(d.fileNameIndex)
-	if n == 0 {
-		return nil
-	}
-
-	n--
-	for what, got := range map[string]int{
-		"boundaries":        len(d.boundaries) - 1,
-		"branch masks":      len(d.fileBranchMasks),
-		"doc section index": len(d.docSectionsIndex) - 1,
-		"newlines index":    len(d.newlinesIndex) - 1,
-	} {
-		if got != n {
-			return fmt.Errorf("got %s %d, want %d", what, got, n)
-		}
-	}
-	return nil
-}
-
-func (d *indexData) readContents(i uint32) ([]byte, error) {
-	return d.readSectionBlob(simpleSection{
-		off: d.boundariesStart + d.boundaries[i],
-		sz:  d.boundaries[i+1] - d.boundaries[i],
-	})
-}
-
-func (d *indexData) readContentSlice(off uint32, sz uint32) ([]byte, error) {
-	// TODO(hanwen): cap result if it is at the end of the content
-	// section.
-	return d.readSectionBlob(simpleSection{
-		off: d.boundariesStart + off,
-		sz:  sz,
-	})
-}
-
-func (d *indexData) readNewlines(i uint32, buf []uint32) ([]uint32, uint32, error) {
-	sec := simpleSection{
-		off: d.newlinesStart + d.newlinesIndex[i],
-		sz:  d.newlinesIndex[i+1] - d.newlinesIndex[i],
-	}
-	blob, err := d.readSectionBlob(sec)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	return fromSizedDeltas(blob, buf), sec.sz, nil
-}
-
-func (d *indexData) readDocSections(i uint32, buf []DocumentSection) ([]DocumentSection, uint32, error) {
-	sec := simpleSection{
-		off: d.docSectionsStart + d.docSectionsIndex[i],
-		sz:  d.docSectionsIndex[i+1] - d.docSectionsIndex[i],
-	}
-	blob, err := d.readSectionBlob(sec)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	return unmarshalDocSections(blob, buf), sec.sz, nil
-}
-
-// NewSearcher creates a Searcher for a single index file.  Search
-// results coming from this searcher are valid only for the lifetime
-// of the Searcher itself, ie. []byte members should be copied into
-// fresh buffers if the result is to survive closing the shard.
-func NewSearcher(r IndexFile) (Searcher, error) {
-	rd := &reader{r: r}
-
-	var toc indexTOC
-	if err := rd.readTOC(&toc); err != nil {
-		return nil, err
-	}
-	indexData, err := rd.readIndexData(&toc)
-	if err != nil {
-		return nil, err
-	}
-	indexData.file = r
-	return indexData, nil
-}
-
-// ReadMetadata returns the metadata of index shard without reading
-// the index data. The IndexFile is not closed.
-func ReadMetadata(inf IndexFile) (*Repository, *IndexMetadata, error) {
-	rd := &reader{r: inf}
-	var toc indexTOC
-	if err := rd.readTOC(&toc); err != nil {
-		return nil, nil, err
-	}
-
-	var md IndexMetadata
-	if err := rd.readJSON(&md, &toc.metaData); err != nil {
-		return nil, nil, err
-	}
-
-	var repo Repository
-	if err := rd.readJSON(&repo, &toc.repoMetaData); err != nil {
-		return nil, nil, err
-	}
-
-	return &repo, &md, nil
-}
diff --git a/read_test.go b/read_test.go
deleted file mode 100644
index b6b57cd..0000000
--- a/read_test.go
+++ /dev/null
@@ -1,178 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
-	"reflect"
-	"testing"
-)
-
-var update = flag.Bool("update", false, "update the golden files of this test")
-
-func TestReadWrite(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	if err := b.AddFile("filename", []byte("abcde")); err != nil {
-		t.Fatalf("AddFile: %v", err)
-	}
-
-	var buf bytes.Buffer
-	b.Write(&buf)
-	f := &memSeeker{buf.Bytes()}
-
-	r := reader{r: f}
-
-	var toc indexTOC
-	err = r.readTOC(&toc)
-
-	if err != nil {
-		t.Errorf("got read error %v", err)
-	}
-	if toc.fileContents.data.sz != 5 {
-		t.Errorf("got contents size %d, want 5", toc.fileContents.data.sz)
-	}
-
-	data, err := r.readIndexData(&toc)
-	if err != nil {
-		t.Fatalf("readIndexData: %v", err)
-	}
-	if got := data.fileName(0); string(got) != "filename" {
-		t.Errorf("got filename %q, want %q", got, "filename")
-	}
-
-	if len(data.ngrams) != 3 {
-		t.Fatalf("got ngrams %v, want 3 ngrams", data.ngrams)
-	}
-
-	if _, ok := data.ngrams[stringToNGram("bcq")]; ok {
-		t.Errorf("found ngram bcd in %v", data.ngrams)
-	}
-}
-
-func TestReadWriteNames(t *testing.T) {
-	b, err := NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	if err := b.AddFile("abCd", []byte("")); err != nil {
-		t.Fatalf("AddFile: %v", err)
-	}
-
-	var buf bytes.Buffer
-	b.Write(&buf)
-	f := &memSeeker{buf.Bytes()}
-
-	r := reader{r: f}
-
-	var toc indexTOC
-	if err := r.readTOC(&toc); err != nil {
-		t.Errorf("got read error %v", err)
-	}
-	if toc.fileNames.data.sz != 4 {
-		t.Errorf("got contents size %d, want 4", toc.fileNames.data.sz)
-	}
-
-	data, err := r.readIndexData(&toc)
-	if err != nil {
-		t.Fatalf("readIndexData: %v", err)
-	}
-	if !reflect.DeepEqual([]uint32{0, 4}, data.fileNameIndex) {
-		t.Errorf("got index %v, want {0,4}", data.fileNameIndex)
-	}
-	if got := data.fileNameNgrams[stringToNGram("bCd")]; !reflect.DeepEqual(got, []uint32{1}) {
-		t.Errorf("got trigram bcd at bits %v, want sz 2", data.fileNameNgrams)
-	}
-}
-
-func TestBackwardsCompat(t *testing.T) {
-	if *update {
-		b, err := NewIndexBuilder(nil)
-		if err != nil {
-			t.Fatalf("NewIndexBuilder: %v", err)
-		}
-
-		if err := b.AddFile("filename", []byte("abcde")); err != nil {
-			t.Fatalf("AddFile: %v", err)
-		}
-
-		var buf bytes.Buffer
-		b.Write(&buf)
-
-		outname := fmt.Sprintf("testdata/backcompat/new_v%d.%05d.zoekt", IndexFormatVersion, 0)
-		t.Log("writing new file", outname)
-
-		err = os.WriteFile(outname, buf.Bytes(), 0644)
-		if err != nil {
-			t.Fatalf("Creating output file: %v", err)
-		}
-	}
-
-	compatibleFiles, err := fs.Glob(os.DirFS("."), "testdata/backcompat/*.zoekt")
-	if err != nil {
-		t.Fatalf("fs.Glob: %v", err)
-	}
-
-	for _, fname := range compatibleFiles {
-		t.Run(path.Base(fname),
-			func(t *testing.T) {
-				f, err := os.Open(fname)
-				if err != nil {
-					t.Fatal("os.Open", err)
-				}
-				idx, err := NewIndexFile(f)
-				if err != nil {
-					t.Fatal("NewIndexFile", err)
-				}
-				r := reader{r: idx}
-
-				var toc indexTOC
-				err = r.readTOC(&toc)
-
-				if err != nil {
-					t.Errorf("got read error %v", err)
-				}
-				if toc.fileContents.data.sz != 5 {
-					t.Errorf("got contents size %d, want 5", toc.fileContents.data.sz)
-				}
-
-				data, err := r.readIndexData(&toc)
-				if err != nil {
-					t.Fatalf("readIndexData: %v", err)
-				}
-				if got := data.fileName(0); string(got) != "filename" {
-					t.Errorf("got filename %q, want %q", got, "filename")
-				}
-
-				if len(data.ngrams) != 3 {
-					t.Fatalf("got ngrams %v, want 3 ngrams", data.ngrams)
-				}
-
-				if _, ok := data.ngrams[stringToNGram("bcq")]; ok {
-					t.Errorf("found ngram bcd in %v", data.ngrams)
-				}
-			},
-		)
-	}
-}
diff --git a/section.go b/section.go
deleted file mode 100644
index 90e4e94..0000000
--- a/section.go
+++ /dev/null
@@ -1,183 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"encoding/binary"
-	"io"
-	"log"
-)
-
-var _ = log.Println
-
-// writer is an io.Writer that keeps track of errors and offsets
-type writer struct {
-	err error
-	w   io.Writer
-	off uint32
-}
-
-func (w *writer) Write(b []byte) error {
-	if w.err != nil {
-		return w.err
-	}
-
-	var n int
-	n, w.err = w.w.Write(b)
-	w.off += uint32(n)
-	return w.err
-}
-
-func (w *writer) Off() uint32 { return w.off }
-
-func (w *writer) B(b byte) {
-	s := []byte{b}
-	w.Write(s)
-}
-
-func (w *writer) U32(n uint32) {
-	var enc [4]byte
-	binary.BigEndian.PutUint32(enc[:], n)
-	w.Write(enc[:])
-}
-
-func (w *writer) U64(n uint64) {
-	var enc [8]byte
-	binary.BigEndian.PutUint64(enc[:], n)
-	w.Write(enc[:])
-}
-
-func (w *writer) Varint(n uint32) {
-	var enc [8]byte
-	m := binary.PutUvarint(enc[:], uint64(n))
-	w.Write(enc[:m])
-}
-
-func (w *writer) String(s string) {
-	b := []byte(s)
-	w.Varint(uint32(len(b)))
-	w.Write(b)
-}
-
-func (s *simpleSection) start(w *writer) {
-	s.off = w.Off()
-}
-
-func (s *simpleSection) end(w *writer) {
-	s.sz = w.Off() - s.off
-}
-
-// section is a range of bytes in the index file.
-type section interface {
-	read(*reader) error
-	write(*writer)
-	kind() sectionKind // simple or complex, used in serialization
-}
-
-type sectionKind int
-
-const (
-	sectionKindSimple  sectionKind = 0
-	sectionKindComplex sectionKind = 1
-)
-
-// simpleSection is a simple range of bytes.
-type simpleSection struct {
-	off uint32
-	sz  uint32
-}
-
-func (s *simpleSection) kind() sectionKind {
-	return sectionKindSimple
-}
-
-func (s *simpleSection) read(r *reader) error {
-	var err error
-	s.off, err = r.U32()
-	if err != nil {
-		return err
-	}
-	s.sz, err = r.U32()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *simpleSection) write(w *writer) {
-	w.U32(s.off)
-	w.U32(s.sz)
-}
-
-// compoundSection is a range of bytes containg a list of variable
-// sized items.
-type compoundSection struct {
-	data simpleSection
-
-	offsets []uint32
-	index   simpleSection
-}
-
-func (s *compoundSection) kind() sectionKind {
-	return sectionKindComplex
-}
-
-func (s *compoundSection) start(w *writer) {
-	s.data.start(w)
-}
-
-func (s *compoundSection) end(w *writer) {
-	s.data.end(w)
-	s.index.start(w)
-	for _, o := range s.offsets {
-		w.U32(o)
-	}
-	s.index.end(w)
-}
-
-func (s *compoundSection) addItem(w *writer, item []byte) {
-	s.offsets = append(s.offsets, w.Off())
-	w.Write(item)
-}
-
-func (s *compoundSection) write(w *writer) {
-	s.data.write(w)
-	s.index.write(w)
-}
-
-func (s *compoundSection) read(r *reader) error {
-	if err := s.data.read(r); err != nil {
-		return err
-	}
-	if err := s.index.read(r); err != nil {
-		return err
-	}
-	var err error
-	s.offsets, err = readSectionU32(r.r, s.index)
-	return err
-}
-
-// relativeIndex returns the relative offsets of the items (first
-// element is 0), plus a final marking the end of the last item.
-func (s *compoundSection) relativeIndex() []uint32 {
-	ri := make([]uint32, 0, len(s.offsets)+1)
-	for _, o := range s.offsets {
-		ri = append(ri, o-s.offsets[0])
-	}
-	if len(s.offsets) > 0 {
-		ri = append(ri, s.data.sz)
-	}
-	return ri
-}
diff --git a/section_test.go b/section_test.go
deleted file mode 100644
index 19ed111..0000000
--- a/section_test.go
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestDeltas(t *testing.T) {
-	in := []uint32{1, 72, 0xfff}
-	out := toSizedDeltas(in)
-	round := fromSizedDeltas(out, nil)
-	if !reflect.DeepEqual(in, round) {
-		t.Errorf("got %v, want %v", round, in)
-	}
-}
diff --git a/shards/shards.go b/shards/shards.go
deleted file mode 100644
index 41afe40..0000000
--- a/shards/shards.go
+++ /dev/null
@@ -1,548 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package shards
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"os"
-	"runtime"
-	"runtime/debug"
-	"sort"
-	"time"
-
-	"golang.org/x/net/trace"
-	"golang.org/x/sync/semaphore"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/query"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	metricShardsLoaded = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "zoekt_shards_loaded",
-		Help: "The number of shards currently loaded",
-	})
-	metricShardsLoadedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_shards_loaded_total",
-		Help: "The total number of shards loaded",
-	})
-	metricShardsLoadFailedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_shards_load_failed_total",
-		Help: "The total number of shard loads that failed",
-	})
-
-	metricSearchRunning = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "zoekt_search_running",
-		Help: "The number of concurrent search requests running",
-	})
-	metricSearchShardRunning = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "zoekt_search_shard_running",
-		Help: "The number of concurrent search requests in a shard running",
-	})
-	metricSearchFailedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_failed_total",
-		Help: "The total number of search requests that failed",
-	})
-	metricSearchDuration = promauto.NewHistogram(prometheus.HistogramOpts{
-		Name:    "zoekt_search_duration_seconds",
-		Help:    "The duration a search request took in seconds",
-		Buckets: prometheus.DefBuckets, // DefBuckets good for service timings
-	})
-
-	// A Counter per Stat. Name should match field in zoekt.Stats.
-	metricSearchContentBytesLoadedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_content_loaded_bytes_total",
-		Help: "Total amount of I/O for reading contents",
-	})
-	metricSearchIndexBytesLoadedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_index_loaded_bytes_total",
-		Help: "Total amount of I/O for reading from index",
-	})
-	metricSearchCrashesTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_crashes_total",
-		Help: "Total number of search shards that had a crash",
-	})
-	metricSearchFileCountTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_file_count_total",
-		Help: "Total number of files containing a match",
-	})
-	metricSearchShardFilesConsideredTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_shard_files_considered_total",
-		Help: "Total number of files in shards that we considered",
-	})
-	metricSearchFilesConsideredTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_files_considered_total",
-		Help: "Total files that we evaluated. Equivalent to files for which all atom matches (including negations) evaluated to true",
-	})
-	metricSearchFilesLoadedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_files_loaded_total",
-		Help: "Total files for which we loaded file content to verify substring matches",
-	})
-	metricSearchFilesSkippedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_files_skipped_total",
-		Help: "Total candidate files whose contents weren't examined because we gathered enough matches",
-	})
-	metricSearchShardsSkippedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_shards_skipped_total",
-		Help: "Total shards that we did not process because a query was canceled",
-	})
-	metricSearchMatchCountTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_match_count_total",
-		Help: "Total number of non-overlapping matches",
-	})
-	metricSearchNgramMatchesTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zoekt_search_ngram_matches_total",
-		Help: "Total number of candidate matches as a result of searching ngrams",
-	})
-
-	metricListRunning = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "zoekt_list_running",
-		Help: "The number of concurrent list requests running",
-	})
-	metricListShardRunning = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "zoekt_list_shard_running",
-		Help: "The number of concurrent list requests in a shard running",
-	})
-)
-
-type rankedShard struct {
-	zoekt.Searcher
-	rank uint16
-}
-
-type shardedSearcher struct {
-	// Limit the number of parallel queries. Since searching is
-	// CPU bound, we can't do better than #CPU queries in
-	// parallel.  If we do so, we just create more memory
-	// pressure.
-	throttle *semaphore.Weighted
-	capacity int64
-
-	shards map[string]rankedShard
-
-	rankedVersion uint64
-	ranked        []rankedShard
-}
-
-func newShardedSearcher(n int64) *shardedSearcher {
-	ss := &shardedSearcher{
-		shards:   make(map[string]rankedShard),
-		throttle: semaphore.NewWeighted(n),
-		capacity: n,
-	}
-	return ss
-}
-
-// NewDirectorySearcher returns a searcher instance that loads all
-// shards corresponding to a glob into memory.
-func NewDirectorySearcher(dir string) (zoekt.Searcher, error) {
-	ss := newShardedSearcher(int64(runtime.GOMAXPROCS(0)))
-	tl := &loader{
-		ss: ss,
-	}
-	dw, err := NewDirectoryWatcher(dir, tl)
-	if err != nil {
-		return nil, err
-	}
-
-	return &directorySearcher{
-		Searcher:         ss,
-		directoryWatcher: dw,
-	}, nil
-}
-
-type directorySearcher struct {
-	zoekt.Searcher
-
-	directoryWatcher *DirectoryWatcher
-}
-
-func (s *directorySearcher) Close() {
-	// We need to Stop directoryWatcher first since it calls load/unload on
-	// Searcher.
-	s.directoryWatcher.Stop()
-	s.Searcher.Close()
-}
-
-type loader struct {
-	ss *shardedSearcher
-}
-
-func (tl *loader) load(key string) {
-	shard, err := loadShard(key)
-	if err != nil {
-		metricShardsLoadFailedTotal.Inc()
-		log.Printf("reloading: %s, err %v ", key, err)
-		return
-	}
-
-	metricShardsLoadedTotal.Inc()
-	tl.ss.replace(key, shard)
-}
-
-func (tl *loader) drop(key string) {
-	tl.ss.replace(key, nil)
-}
-
-func (ss *shardedSearcher) String() string {
-	return "shardedSearcher"
-}
-
-// Close closes references to open files. It may be called only once.
-func (ss *shardedSearcher) Close() {
-	ss.lock()
-	defer ss.unlock()
-	for _, s := range ss.shards {
-		s.Close()
-	}
-	ss.shards = make(map[string]rankedShard)
-}
-
-func (ss *shardedSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (sr *zoekt.SearchResult, err error) {
-	tr := trace.New("shardedSearcher.Search", "")
-	tr.LazyLog(q, true)
-	tr.LazyPrintf("opts: %+v", opts)
-	overallStart := time.Now()
-	metricSearchRunning.Inc()
-	defer func() {
-		metricSearchRunning.Dec()
-		metricSearchDuration.Observe(time.Since(overallStart).Seconds())
-		if sr != nil {
-			metricSearchContentBytesLoadedTotal.Add(float64(sr.Stats.ContentBytesLoaded))
-			metricSearchIndexBytesLoadedTotal.Add(float64(sr.Stats.IndexBytesLoaded))
-			metricSearchCrashesTotal.Add(float64(sr.Stats.Crashes))
-			metricSearchFileCountTotal.Add(float64(sr.Stats.FileCount))
-			metricSearchShardFilesConsideredTotal.Add(float64(sr.Stats.ShardFilesConsidered))
-			metricSearchFilesConsideredTotal.Add(float64(sr.Stats.FilesConsidered))
-			metricSearchFilesLoadedTotal.Add(float64(sr.Stats.FilesLoaded))
-			metricSearchFilesSkippedTotal.Add(float64(sr.Stats.FilesSkipped))
-			metricSearchShardsSkippedTotal.Add(float64(sr.Stats.ShardsSkipped))
-			metricSearchMatchCountTotal.Add(float64(sr.Stats.MatchCount))
-			metricSearchNgramMatchesTotal.Add(float64(sr.Stats.NgramMatches))
-
-			tr.LazyPrintf("num files: %d", len(sr.Files))
-			tr.LazyPrintf("stats: %+v", sr.Stats)
-		}
-		if err != nil {
-			metricSearchFailedTotal.Inc()
-
-			tr.LazyPrintf("error: %v", err)
-			tr.SetError()
-		}
-		tr.Finish()
-	}()
-
-	start := time.Now()
-
-	aggregate := &zoekt.SearchResult{
-		RepoURLs:      map[string]string{},
-		LineFragments: map[string]string{},
-	}
-
-	// This critical section is large, but we don't want to deal with
-	// searches on shards that have just been closed.
-	if err := ss.rlock(ctx); err != nil {
-		return aggregate, err
-	}
-	defer ss.runlock()
-	tr.LazyPrintf("acquired lock")
-	aggregate.Wait = time.Since(start)
-	start = time.Now()
-
-	shards := ss.getShards()
-	all := make(chan shardResult, len(shards))
-
-	var childCtx context.Context
-	var cancel context.CancelFunc
-	if opts.MaxWallTime == 0 {
-		childCtx, cancel = context.WithCancel(ctx)
-	} else {
-		childCtx, cancel = context.WithTimeout(ctx, opts.MaxWallTime)
-	}
-
-	defer cancel()
-
-	// For each query, throttle the number of parallel
-	// actions. Since searching is mostly CPU bound, we limit the
-	// number of parallel searches. This reduces the peak working
-	// set, which hopefully stops https://cs.bazel.build from crashing
-	// when looking for the string "com".
-	feeder := make(chan zoekt.Searcher, len(shards))
-	for _, s := range shards {
-		feeder <- s
-	}
-	close(feeder)
-	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
-		go func() {
-			for s := range feeder {
-				searchOneShard(childCtx, s, q, opts, all)
-			}
-		}()
-	}
-
-	for range shards {
-		r := <-all
-		if r.err != nil {
-			return nil, r.err
-		}
-		aggregate.Files = append(aggregate.Files, r.sr.Files...)
-		aggregate.Stats.Add(r.sr.Stats)
-
-		if len(r.sr.Files) > 0 {
-			for k, v := range r.sr.RepoURLs {
-				aggregate.RepoURLs[k] = v
-			}
-			for k, v := range r.sr.LineFragments {
-				aggregate.LineFragments[k] = v
-			}
-		}
-
-		if cancel != nil && opts.TotalMaxMatchCount > 0 && aggregate.Stats.MatchCount > opts.TotalMaxMatchCount {
-			cancel()
-			cancel = nil
-		}
-	}
-
-	zoekt.SortFilesByScore(aggregate.Files)
-	if max := opts.MaxDocDisplayCount; max > 0 && len(aggregate.Files) > max {
-		aggregate.Files = aggregate.Files[:max]
-	}
-	for i := range aggregate.Files {
-		copySlice(&aggregate.Files[i].Content)
-		copySlice(&aggregate.Files[i].Checksum)
-		for l := range aggregate.Files[i].LineMatches {
-			copySlice(&aggregate.Files[i].LineMatches[l].Line)
-		}
-	}
-
-	aggregate.Duration = time.Since(start)
-	return aggregate, nil
-}
-
-func copySlice(src *[]byte) {
-	dst := make([]byte, len(*src))
-	copy(dst, *src)
-	*src = dst
-}
-
-type shardResult struct {
-	sr  *zoekt.SearchResult
-	err error
-}
-
-func searchOneShard(ctx context.Context, s zoekt.Searcher, q query.Q, opts *zoekt.SearchOptions, sink chan shardResult) {
-	metricSearchShardRunning.Inc()
-	defer func() {
-		metricSearchShardRunning.Dec()
-		if r := recover(); r != nil {
-			log.Printf("crashed shard: %s: %s, %s", s.String(), r, debug.Stack())
-
-			var r zoekt.SearchResult
-			r.Stats.Crashes = 1
-			sink <- shardResult{&r, nil}
-		}
-	}()
-
-	ms, err := s.Search(ctx, q, opts)
-	sink <- shardResult{ms, err}
-}
-
-func (ss *shardedSearcher) List(ctx context.Context, r query.Q) (rl *zoekt.RepoList, err error) {
-	tr := trace.New("shardedSearcher.List", "")
-	tr.LazyLog(r, true)
-	metricListRunning.Inc()
-	defer func() {
-		metricListRunning.Dec()
-		if rl != nil {
-			tr.LazyPrintf("repos size: %d", len(rl.Repos))
-			tr.LazyPrintf("crashes: %d", rl.Crashes)
-		}
-		if err != nil {
-			tr.LazyPrintf("error: %v", err)
-			tr.SetError()
-		}
-		tr.Finish()
-	}()
-
-	type res struct {
-		rl  *zoekt.RepoList
-		err error
-	}
-
-	if err := ss.rlock(ctx); err != nil {
-		return nil, err
-	}
-	defer ss.runlock()
-	tr.LazyPrintf("acquired lock")
-
-	shards := ss.getShards()
-	shardCount := len(shards)
-	all := make(chan res, shardCount)
-	tr.LazyPrintf("shardCount: %d", len(shards))
-
-	for _, s := range shards {
-		go func(s zoekt.Searcher) {
-			metricListShardRunning.Inc()
-			defer func() {
-				metricListShardRunning.Dec()
-				if r := recover(); r != nil {
-					all <- res{
-						&zoekt.RepoList{Crashes: 1}, nil,
-					}
-				}
-			}()
-			ms, err := s.List(ctx, r)
-			all <- res{ms, err}
-		}(s.Searcher)
-	}
-
-	crashes := 0
-	uniq := map[string]*zoekt.RepoListEntry{}
-
-	for i := 0; i < shardCount; i++ {
-		r := <-all
-		if r.err != nil {
-			return nil, r.err
-		}
-		crashes += r.rl.Crashes
-		for _, r := range r.rl.Repos {
-			prev, ok := uniq[r.Repository.Name]
-			if !ok {
-				cp := *r
-				uniq[r.Repository.Name] = &cp
-			} else {
-				prev.Stats.Add(&r.Stats)
-			}
-		}
-	}
-
-	aggregate := make([]*zoekt.RepoListEntry, 0, len(uniq))
-	for _, v := range uniq {
-		aggregate = append(aggregate, v)
-	}
-	return &zoekt.RepoList{
-		Repos:   aggregate,
-		Crashes: crashes,
-	}, nil
-}
-
-func (s *shardedSearcher) rlock(ctx context.Context) error {
-	return s.throttle.Acquire(ctx, 1)
-}
-
-// getShards returns the currently loaded shards. The shards must be accessed
-// under a rlock call. The shards are sorted by decreasing rank and should not
-// be mutated.
-func (s *shardedSearcher) getShards() []rankedShard {
-	if len(s.ranked) > 0 {
-		return s.ranked
-	}
-
-	var res []rankedShard
-	for _, sh := range s.shards {
-		res = append(res, sh)
-	}
-	sort.Slice(res, func(i, j int) bool {
-		return res[i].rank > res[j].rank
-	})
-
-	// Cache ranked. We currently hold a read lock, so start a goroutine which
-	// acquires a write lock to update. Use requiredVersion to ensure our
-	// cached slice is still current after acquiring the write lock.
-	go func(ranked []rankedShard, requiredVersion uint64) {
-		s.lock()
-		if s.rankedVersion == requiredVersion {
-			s.ranked = ranked
-		}
-		s.unlock()
-	}(res, s.rankedVersion)
-
-	return res
-}
-
-func (s *shardedSearcher) runlock() {
-	s.throttle.Release(1)
-}
-
-func (s *shardedSearcher) lock() {
-	// won't error since context.Background won't expire
-	_ = s.throttle.Acquire(context.Background(), s.capacity)
-}
-
-func (s *shardedSearcher) unlock() {
-	s.throttle.Release(s.capacity)
-}
-
-func shardRank(s zoekt.Searcher) uint16 {
-	q := query.Repo{}
-	result, err := s.List(context.Background(), &q)
-	if err != nil {
-		return 0
-	}
-	if len(result.Repos) == 0 {
-		return 0
-	}
-	return result.Repos[0].Repository.Rank
-}
-
-func (s *shardedSearcher) replace(key string, shard zoekt.Searcher) {
-	var rank uint16
-	if shard != nil {
-		rank = shardRank(shard)
-	}
-
-	s.lock()
-	defer s.unlock()
-	old := s.shards[key]
-	if old.Searcher != nil {
-		old.Close()
-	}
-
-	if shard == nil {
-		delete(s.shards, key)
-	} else {
-		s.shards[key] = rankedShard{
-			rank:     rank,
-			Searcher: shard,
-		}
-	}
-	s.rankedVersion++
-	s.ranked = nil
-
-	metricShardsLoaded.Set(float64(len(s.shards)))
-}
-
-func loadShard(fn string) (zoekt.Searcher, error) {
-	f, err := os.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-
-	iFile, err := zoekt.NewIndexFile(f)
-	if err != nil {
-		return nil, err
-	}
-	s, err := zoekt.NewSearcher(iFile)
-	if err != nil {
-		iFile.Close()
-		return nil, fmt.Errorf("NewSearcher(%s): %v", fn, err)
-	}
-
-	return s, nil
-}
diff --git a/shards/shards_test.go b/shards/shards_test.go
deleted file mode 100644
index 3df2025..0000000
--- a/shards/shards_test.go
+++ /dev/null
@@ -1,228 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package shards
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"log"
-	"os"
-	"runtime"
-	"testing"
-	"time"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/query"
-)
-
-type crashSearcher struct{}
-
-func (s *crashSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) {
-	panic("search")
-}
-
-func (s *crashSearcher) List(ctx context.Context, q query.Q) (*zoekt.RepoList, error) {
-	panic("list")
-}
-
-func (s *crashSearcher) Stats() (*zoekt.RepoStats, error) {
-	return &zoekt.RepoStats{}, nil
-}
-
-func (s *crashSearcher) Close() {}
-
-func (s *crashSearcher) String() string { return "crashSearcher" }
-
-func TestCrashResilience(t *testing.T) {
-	out := &bytes.Buffer{}
-	log.SetOutput(out)
-	defer log.SetOutput(os.Stderr)
-	ss := newShardedSearcher(2)
-	ss.shards = map[string]rankedShard{
-		"x": {Searcher: &crashSearcher{}},
-	}
-
-	q := &query.Substring{Pattern: "hoi"}
-	opts := &zoekt.SearchOptions{}
-	if res, err := ss.Search(context.Background(), q, opts); err != nil {
-		t.Fatalf("Search: %v", err)
-	} else if res.Stats.Crashes != 1 {
-		t.Errorf("got stats %#v, want crashes = 1", res.Stats)
-	}
-
-	if res, err := ss.List(context.Background(), q); err != nil {
-		t.Fatalf("List: %v", err)
-	} else if res.Crashes != 1 {
-		t.Errorf("got result %#v, want crashes = 1", res)
-	}
-}
-
-type rankSearcher struct {
-	rank uint16
-}
-
-func (s *rankSearcher) Close() {
-}
-
-func (s *rankSearcher) String() string {
-	return ""
-}
-
-func (s *rankSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) {
-	select {
-	case <-ctx.Done():
-		return &zoekt.SearchResult{}, nil
-	default:
-	}
-
-	// Ugly, but without sleep it's too fast, and we can't
-	// simulate the cutoff.
-	time.Sleep(time.Millisecond)
-	return &zoekt.SearchResult{
-		Files: []zoekt.FileMatch{
-			{
-				FileName: fmt.Sprintf("f%d", s.rank),
-				Score:    float64(s.rank),
-			},
-		},
-		Stats: zoekt.Stats{
-			MatchCount: 1,
-		},
-	}, nil
-}
-
-func (s *rankSearcher) List(ctx context.Context, q query.Q) (*zoekt.RepoList, error) {
-	return &zoekt.RepoList{
-		Repos: []*zoekt.RepoListEntry{
-			{Repository: zoekt.Repository{Rank: s.rank}},
-		},
-	}, nil
-}
-
-func TestOrderByShard(t *testing.T) {
-	ss := newShardedSearcher(1)
-
-	n := 10 * runtime.GOMAXPROCS(0)
-	for i := 0; i < n; i++ {
-		ss.replace(fmt.Sprintf("shard%d", i),
-			&rankSearcher{
-				rank: uint16(i),
-			})
-	}
-
-	if res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &zoekt.SearchOptions{}); err != nil {
-		t.Errorf("Search: %v", err)
-	} else if len(res.Files) != n {
-		t.Fatalf("empty options: got %d results, want %d", len(res.Files), n)
-	}
-
-	opts := zoekt.SearchOptions{
-		TotalMaxMatchCount: 3,
-	}
-	res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &opts)
-	if err != nil {
-		t.Errorf("Search: %v", err)
-	}
-
-	if len(res.Files) < opts.TotalMaxMatchCount {
-		t.Errorf("got %d results, want %d", len(res.Files), opts.TotalMaxMatchCount)
-	}
-	if len(res.Files) == n {
-		t.Errorf("got %d results, want < %d", len(res.Files), n)
-	}
-	for i, f := range res.Files {
-		rev := n - 1 - i
-		want := fmt.Sprintf("f%d", rev)
-		got := f.FileName
-
-		if got != want {
-			t.Logf("%d: got %q, want %q", i, got, want)
-		}
-	}
-}
-
-type memSeeker struct {
-	data []byte
-}
-
-func (s *memSeeker) Name() string {
-	return "memseeker"
-}
-
-func (s *memSeeker) Close() {}
-func (s *memSeeker) Read(off, sz uint32) ([]byte, error) {
-	return s.data[off : off+sz], nil
-}
-
-func (s *memSeeker) Size() (uint32, error) {
-	return uint32(len(s.data)), nil
-}
-
-func TestUnloadIndex(t *testing.T) {
-	b, err := zoekt.NewIndexBuilder(nil)
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	for i, d := range []zoekt.Document{{
-		Name:    "filename",
-		Content: []byte("needle needle needle"),
-	}} {
-		if err := b.Add(d); err != nil {
-			t.Fatalf("Add %d: %v", i, err)
-		}
-	}
-
-	var buf bytes.Buffer
-	b.Write(&buf)
-	indexBytes := buf.Bytes()
-	indexFile := &memSeeker{indexBytes}
-	searcher, err := zoekt.NewSearcher(indexFile)
-	if err != nil {
-		t.Fatalf("NewSearcher: %v", err)
-	}
-
-	ss := newShardedSearcher(2)
-	ss.replace("key", searcher)
-
-	var opts zoekt.SearchOptions
-	q := &query.Substring{Pattern: "needle"}
-	res, err := ss.Search(context.Background(), q, &opts)
-	if err != nil {
-		t.Fatalf("Search(%s): %v", q, err)
-	}
-
-	forbidden := byte(29)
-	for i := range indexBytes {
-		// non-ASCII
-		indexBytes[i] = forbidden
-	}
-
-	for _, f := range res.Files {
-		if bytes.Contains(f.Content, []byte{forbidden}) {
-			t.Errorf("found %d in content %q", forbidden, f.Content)
-		}
-		if bytes.Contains(f.Checksum, []byte{forbidden}) {
-			t.Errorf("found %d in checksum %q", forbidden, f.Checksum)
-		}
-
-		for _, l := range f.LineMatches {
-			if bytes.Contains(l.Line, []byte{forbidden}) {
-				t.Errorf("found %d in line %q", forbidden, l.Line)
-			}
-		}
-	}
-}
diff --git a/shards/watcher.go b/shards/watcher.go
deleted file mode 100644
index 62f2067..0000000
--- a/shards/watcher.go
+++ /dev/null
@@ -1,212 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package shards
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"runtime"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-)
-
-type shardLoader interface {
-	// Load a new file. Should be safe for concurrent calls.
-	load(filename string)
-	drop(filename string)
-}
-
-type DirectoryWatcher struct {
-	dir        string
-	timestamps map[string]time.Time
-	loader     shardLoader
-
-	closeOnce sync.Once
-	// quit is closed by Close to signal the directory watcher to stop.
-	quit chan struct{}
-	// stopped is closed once the directory watcher has stopped.
-	stopped chan struct{}
-}
-
-func (sw *DirectoryWatcher) Stop() {
-	sw.closeOnce.Do(func() {
-		close(sw.quit)
-		<-sw.stopped
-	})
-}
-
-func NewDirectoryWatcher(dir string, loader shardLoader) (*DirectoryWatcher, error) {
-	sw := &DirectoryWatcher{
-		dir:        dir,
-		timestamps: map[string]time.Time{},
-		loader:     loader,
-		quit:       make(chan struct{}),
-		stopped:    make(chan struct{}),
-	}
-	if err := sw.scan(); err != nil {
-		return nil, err
-	}
-
-	if err := sw.watch(); err != nil {
-		return nil, err
-	}
-
-	return sw, nil
-}
-
-func (s *DirectoryWatcher) String() string {
-	return fmt.Sprintf("shardWatcher(%s)", s.dir)
-}
-
-func (s *DirectoryWatcher) scan() error {
-	fs, err := filepath.Glob(filepath.Join(s.dir, "*.zoekt"))
-	if err != nil {
-		return err
-	}
-
-	if len(s.timestamps) == 0 && len(fs) == 0 {
-		return fmt.Errorf("directory %s is empty", s.dir)
-	}
-
-	ts := map[string]time.Time{}
-	for _, fn := range fs {
-		fi, err := os.Lstat(fn)
-		if err != nil {
-			continue
-		}
-
-		ts[fn] = fi.ModTime()
-	}
-
-	var toLoad []string
-	for k, mtime := range ts {
-		if t, ok := s.timestamps[k]; !ok || t != mtime {
-			toLoad = append(toLoad, k)
-			s.timestamps[k] = mtime
-		}
-	}
-
-	var toDrop []string
-	// Unload deleted shards.
-	for k := range s.timestamps {
-		if _, ok := ts[k]; !ok {
-			toDrop = append(toDrop, k)
-			delete(s.timestamps, k)
-		}
-	}
-
-	if len(toDrop) > 0 {
-		log.Printf("unloading %d shard(s)", len(toDrop))
-	}
-	for _, t := range toDrop {
-		log.Printf("unloading: %s", filepath.Base(t))
-		s.loader.drop(t)
-	}
-
-	if len(toLoad) == 0 {
-		return nil
-	}
-
-	log.Printf("loading %d shard(s): %s", len(toLoad), humanTruncateList(toLoad, 5))
-
-	// Limit amount of concurrent shard loads.
-	throttle := make(chan struct{}, runtime.GOMAXPROCS(0))
-	lastProgress := time.Now()
-	for i, t := range toLoad {
-		// If taking a while to start-up occasionally give a progress message
-		if time.Since(lastProgress) > 10*time.Second {
-			log.Printf("still need to load %d shards...", len(toLoad)-i)
-			lastProgress = time.Now()
-		}
-
-		throttle <- struct{}{}
-		go func(k string) {
-			s.loader.load(k)
-			<-throttle
-		}(t)
-	}
-	for i := 0; i < cap(throttle); i++ {
-		throttle <- struct{}{}
-	}
-
-	return nil
-}
-
-func humanTruncateList(paths []string, max int) string {
-	sort.Strings(paths)
-	var b strings.Builder
-	for i, p := range paths {
-		if i >= max {
-			fmt.Fprintf(&b, "... %d more", len(paths)-i)
-			break
-		}
-		if i > 0 {
-			b.WriteString(", ")
-		}
-		b.WriteString(filepath.Base(p))
-	}
-	return b.String()
-}
-
-func (s *DirectoryWatcher) watch() error {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		return err
-	}
-	if err := watcher.Add(s.dir); err != nil {
-		return err
-	}
-
-	// intermediate signal channel so if there are multiple watcher.Events we
-	// only call scan once.
-	signal := make(chan struct{}, 1)
-
-	go func() {
-		for {
-			select {
-			case <-watcher.Events:
-				select {
-				case signal <- struct{}{}:
-				default:
-				}
-			case err := <-watcher.Errors:
-				// Ignore ErrEventOverflow since we rely on the presence of events so
-				// safe to ignore.
-				if err != nil && err != fsnotify.ErrEventOverflow {
-					log.Println("watcher error:", err)
-				}
-			case <-s.quit:
-				watcher.Close()
-				close(signal)
-				return
-			}
-		}
-	}()
-
-	go func() {
-		defer close(s.stopped)
-		for range signal {
-			s.scan()
-		}
-	}()
-
-	return nil
-}
diff --git a/shards/watcher_test.go b/shards/watcher_test.go
deleted file mode 100644
index 5aad02f..0000000
--- a/shards/watcher_test.go
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package shards
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-)
-
-type loggingLoader struct {
-	loads chan string
-	drops chan string
-}
-
-func (l *loggingLoader) load(k string) {
-	l.loads <- k
-}
-
-func (l *loggingLoader) drop(k string) {
-	l.drops <- k
-}
-
-func advanceFS() {
-	time.Sleep(10 * time.Millisecond)
-}
-
-func TestDirWatcherUnloadOnce(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	logger := &loggingLoader{
-		loads: make(chan string, 10),
-		drops: make(chan string, 10),
-	}
-	_, err = NewDirectoryWatcher(dir, logger)
-	if err == nil || !strings.Contains(err.Error(), "empty") {
-		t.Fatalf("got %v, want 'empty'", err)
-	}
-
-	shard := filepath.Join(dir, "foo.zoekt")
-	if err := ioutil.WriteFile(shard, []byte("hello"), 0o644); err != nil {
-		t.Fatalf("WriteFile: %v", err)
-	}
-
-	dw, err := NewDirectoryWatcher(dir, logger)
-	if err != nil {
-		t.Fatalf("NewDirectoryWatcher: %v", err)
-	}
-	defer dw.Stop()
-
-	if got := <-logger.loads; got != shard {
-		t.Fatalf("got load event %v, want %v", got, shard)
-	}
-
-	// Must sleep because of FS timestamp resolution.
-	advanceFS()
-	if err := ioutil.WriteFile(shard, []byte("changed"), 0o644); err != nil {
-		t.Fatalf("WriteFile: %v", err)
-	}
-
-	if got := <-logger.loads; got != shard {
-		t.Fatalf("got load event %v, want %v", got, shard)
-	}
-
-	advanceFS()
-	if err := os.Remove(shard); err != nil {
-		t.Fatalf("Remove: %v", err)
-	}
-
-	if got := <-logger.drops; got != shard {
-		t.Fatalf("got drops event %v, want %v", got, shard)
-	}
-
-	advanceFS()
-	if err := ioutil.WriteFile(shard+".bla", []byte("changed"), 0o644); err != nil {
-		t.Fatalf("WriteFile: %v", err)
-	}
-
-	dw.Stop()
-
-	select {
-	case k := <-logger.loads:
-		t.Errorf("spurious load of %q", k)
-	case k := <-logger.drops:
-		t.Errorf("spurious drops of %q", k)
-	default:
-	}
-}
-
-func TestHumanTruncateList(t *testing.T) {
-	paths := []string{
-		"dir/1",
-		"dir/2",
-		"dir/3",
-		"dir/4",
-	}
-
-	assert := func(max int, want string) {
-		got := humanTruncateList(paths, max)
-		if got != want {
-			t.Errorf("unexpected humanTruncateList max=%d.\ngot:  %s\nwant: %s", max, got, want)
-		}
-	}
-
-	assert(1, "1... 3 more")
-	assert(2, "1, 2... 2 more")
-	assert(3, "1, 2, 3... 1 more")
-	assert(4, "1, 2, 3, 4")
-	assert(5, "1, 2, 3, 4")
-}
diff --git a/testdata/backcompat/static_toc_v15.00000.zoekt b/testdata/backcompat/static_toc_v15.00000.zoekt
deleted file mode 100644
index a070892..0000000
--- a/testdata/backcompat/static_toc_v15.00000.zoekt
+++ /dev/null
Binary files differ
diff --git a/toc.go b/toc.go
deleted file mode 100644
index 9eab283..0000000
--- a/toc.go
+++ /dev/null
@@ -1,163 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-// FormatVersion is a version number. It is increased every time the
-// on-disk index format is changed.
-// 5: subrepositories.
-// 6: remove size prefix for posting varint list.
-// 7: move subrepos into Repository struct.
-// 8: move repoMetaData out of indexMetadata
-// 9: use bigendian uint64 for trigrams.
-// 10: sections for rune offsets.
-// 11: file ends in rune offsets.
-// 12: 64-bit branchmasks.
-// 13: content checksums
-// 14: languages
-// 15: rune based symbol sections
-// 16 (TBA): TODO: remove fallback parsing in readTOC
-const IndexFormatVersion = 15
-
-// FeatureVersion is increased if a feature is added that requires reindexing data
-// without changing the format version
-// 2: Rank field for shards.
-// 3: Rank documents within shards
-// 4: Dedup file bugfix
-// 5: Remove max line size limit
-// 6: Include '#' into the LineFragment template
-// 7: Record skip reasons in the index.
-// 8: Record source path in the index.
-// 9: Bump default max file size.
-// 10: Switch to a more flexible TOC format.
-const FeatureVersion = 10
-
-// WriteMinFeatureVersion and ReadMinFeatureVersion constrain forwards and backwards
-// compatibility. For example, if a new way to encode filenameNgrams on disk is
-// added using a new section but the old one is retained, this would only bump
-// FeatureVersion, since the previous version can read the file and ignore the
-// new section, but the index files should be regenerated.
-// When the new encoding is fully rolled out and stable, the section with the old
-// encoding and the associated reader can be removed, and WriteMinFeatureVersion and
-// ReadMinFeatureVersion can be set to the current FeatureVersion, indicating
-// that the reader must handle the new version and that older versions are no
-// longer valid.
-// In this way, compatibility with arbitrary version offsets can be indicated.
-
-// WriteMinFeatureVersion constrains forwards compatibility by emitting files
-// that won't load in zoekt with a FeatureVersion below it.
-const WriteMinFeatureVersion = 10
-
-// ReadMinFeatureVersion constrains backwards compatibility by refusing to
-// load a file with a FeatureVersion below it.
-const ReadMinFeatureVersion = 8
-
-type indexTOC struct {
-	fileContents compoundSection
-	fileNames    compoundSection
-	fileSections compoundSection
-	postings     compoundSection
-	newlines     compoundSection
-	ngramText    simpleSection
-	runeOffsets  simpleSection
-	fileEndRunes simpleSection
-	languages    simpleSection
-
-	branchMasks simpleSection
-	subRepos    simpleSection
-
-	nameNgramText    simpleSection
-	namePostings     compoundSection
-	nameRuneOffsets  simpleSection
-	metaData         simpleSection
-	repoMetaData     simpleSection
-	nameEndRunes     simpleSection
-	contentChecksums simpleSection
-	runeDocSections  simpleSection
-}
-
-func (t *indexTOC) sections() []section {
-	// This old sections list is only needed to maintain backwards compatibility,
-	// and can be removed when a migration to tagged sections is complete.
-	return []section{
-		// This must be first, so it can be reliably read across
-		// file format versions.
-		&t.metaData,
-		&t.repoMetaData,
-		&t.fileContents,
-		&t.fileNames,
-		&t.fileSections,
-		&t.newlines,
-		&t.ngramText,
-		&t.postings,
-		&t.nameNgramText,
-		&t.namePostings,
-		&t.branchMasks,
-		&t.subRepos,
-		&t.runeOffsets,
-		&t.nameRuneOffsets,
-		&t.fileEndRunes,
-		&t.nameEndRunes,
-		&t.contentChecksums,
-		&t.languages,
-		&t.runeDocSections,
-	}
-}
-
-type taggedSection struct {
-	tag string
-	sec section
-}
-
-func (t *indexTOC) sectionsTagged() map[string]section {
-	out := map[string]section{}
-	for _, ent := range t.sectionsTaggedList() {
-		out[ent.tag] = ent.sec
-	}
-	for _, ent := range t.sectionsTaggedCompatibilityList() {
-		out[ent.tag] = ent.sec
-	}
-	return out
-}
-
-func (t *indexTOC) sectionsTaggedList() []taggedSection {
-	return []taggedSection{
-		{"metadata", &t.metaData},
-		{"repoMetaData", &t.repoMetaData},
-		{"fileContents", &t.fileContents},
-		{"fileNames", &t.fileNames},
-		{"fileSections", &t.fileSections},
-		{"newlines", &t.newlines},
-		{"ngramText", &t.ngramText},
-		{"postings", &t.postings},
-		{"nameNgramText", &t.nameNgramText},
-		{"namePostings", &t.namePostings},
-		{"branchMasks", &t.branchMasks},
-		{"subRepos", &t.subRepos},
-		{"runeOffsets", &t.runeOffsets},
-		{"nameRuneOffsets", &t.nameRuneOffsets},
-		{"fileEndRunes", &t.fileEndRunes},
-		{"nameEndRunes", &t.nameEndRunes},
-		{"contentChecksums", &t.contentChecksums},
-		{"languages", &t.languages},
-		{"runeDocSections", &t.runeDocSections},
-	}
-}
-
-// sectionsTaggedCompatibilityList returns a list of sections that will be
-// handled or converted for backwards compatiblity, but aren't written by
-// the current iteration of the indexer.
-func (t *indexTOC) sectionsTaggedCompatibilityList() []taggedSection {
-	return []taggedSection{}
-}
diff --git a/web/api.go b/web/api.go
deleted file mode 100644
index 8eca345..0000000
--- a/web/api.go
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package web
-
-import (
-	"time"
-
-	"github.com/google/zoekt"
-)
-
-type LastInput struct {
-	Query string
-	Num   int
-
-	// If set, focus on the search box.
-	AutoFocus bool
-}
-
-// Result holds the data provided to the search results template.
-type ResultInput struct {
-	Last          LastInput
-	QueryStr      string
-	Query         string
-	Stats         zoekt.Stats
-	Duration      time.Duration
-	FileMatches   []*FileMatch
-	SearchOptions string
-}
-
-// FileMatch holds the per file data provided to search results template
-type FileMatch struct {
-	FileName string
-	Repo     string
-	ResultID string
-	Language string
-	// If this was a duplicate result, this will contain the file
-	// of the first match.
-	DuplicateID string
-
-	Branches []string
-	Matches  []Match
-	URL      string
-}
-
-// Match holds the per line data provided to the search results template
-type Match struct {
-	URL      string
-	FileName string
-	LineNum  int
-
-	Fragments []Fragment
-}
-
-// Fragment holds data of a single contiguous match within in a line
-// for the results template.
-type Fragment struct {
-	Pre   string
-	Match string
-	Post  string
-}
-
-// SearchBoxInput is provided to the SearchBox template.
-type SearchBoxInput struct {
-	Last    LastInput
-	Stats   *zoekt.RepoStats
-	Version string
-	Uptime  time.Duration
-}
-
-// RepoListInput is provided to the RepoList template.
-type RepoListInput struct {
-	Last  LastInput
-	Stats zoekt.RepoStats
-	Repos []Repository
-}
-
-// Branch holds the metadata for a indexed branch.
-type Branch struct {
-	Name    string
-	Version string
-	URL     string
-}
-
-// Repository holds the metadata for an indexed repository.
-type Repository struct {
-	Name      string
-	URL       string
-	IndexTime time.Time
-	Branches  []Branch
-	Files     int64
-
-	// Total amount of content bytes.
-	Size int64
-}
-
-// PrintInput is provided to the server.Print template.
-type PrintInput struct {
-	Repo, Name string
-	Lines      []string
-	Last       LastInput
-}
diff --git a/web/e2e_test.go b/web/e2e_test.go
deleted file mode 100644
index 479441b..0000000
--- a/web/e2e_test.go
+++ /dev/null
@@ -1,442 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package web
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/query"
-)
-
-// TODO(hanwen): cut & paste from ../ . Should create internal test
-// util package.
-type memSeeker struct {
-	data []byte
-}
-
-func (s *memSeeker) Close() {}
-func (s *memSeeker) Read(off, sz uint32) ([]byte, error) {
-	return s.data[off : off+sz], nil
-}
-
-func (s *memSeeker) Size() (uint32, error) {
-	return uint32(len(s.data)), nil
-}
-
-func (s *memSeeker) Name() string {
-	return "memSeeker"
-}
-
-func searcherForTest(t *testing.T, b *zoekt.IndexBuilder) zoekt.Searcher {
-	var buf bytes.Buffer
-	b.Write(&buf)
-	f := &memSeeker{buf.Bytes()}
-
-	searcher, err := zoekt.NewSearcher(f)
-	if err != nil {
-		t.Fatalf("NewSearcher: %v", err)
-	}
-
-	return searcher
-}
-
-func TestBasic(t *testing.T) {
-	b, err := zoekt.NewIndexBuilder(&zoekt.Repository{
-		Name:                 "name",
-		URL:                  "repo-url",
-		CommitURLTemplate:    "{{.Version}}",
-		FileURLTemplate:      "file-url",
-		LineFragmentTemplate: "#line",
-		Branches:             []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
-	})
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-	if err := b.Add(zoekt.Document{
-		Name:    "f2",
-		Content: []byte("to carry water in the no later bla"),
-		// ------------- 0123456789012345678901234567890123
-		// ------------- 0         1         2         3
-		Branches: []string{"master"},
-	}); err != nil {
-		t.Fatalf("Add: %v", err)
-	}
-
-	s := searcherForTest(t, b)
-	srv := Server{
-		Searcher: s,
-		Top:      Top,
-		HTML:     true,
-	}
-
-	mux, err := NewMux(&srv)
-	if err != nil {
-		t.Fatalf("NewMux: %v", err)
-	}
-
-	ts := httptest.NewServer(mux)
-	defer ts.Close()
-
-	nowStr := time.Now().Format("Jan 02, 2006 15:04")
-	for req, needles := range map[string][]string{
-		"/": {"from 1 repositories"},
-		"/search?q=water": {
-			"href=\"file-url#line",
-			"carry <b>water</b>",
-		},
-		"/search?q=r:": {
-			"1234\">master",
-			"Found 1 repositories",
-			nowStr,
-			"repo-url\">name",
-			"1 files (36)",
-		},
-		"/search?q=magic": {
-			`value=magic`,
-		},
-		"/robots.txt": {
-			"disallow: /search",
-		},
-	} {
-		checkNeedles(t, ts, req, needles)
-	}
-}
-
-func TestPrint(t *testing.T) {
-	b, err := zoekt.NewIndexBuilder(&zoekt.Repository{
-		Name:                 "name",
-		URL:                  "repo-url",
-		CommitURLTemplate:    "{{.Version}}",
-		FileURLTemplate:      "file-url",
-		LineFragmentTemplate: "line",
-		Branches:             []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
-	})
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-	if err := b.Add(zoekt.Document{
-		Name:     "f2",
-		Content:  []byte("to carry water in the no later bla"),
-		Branches: []string{"master"},
-	}); err != nil {
-		t.Fatalf("Add: %v", err)
-	}
-
-	if err := b.Add(zoekt.Document{
-		Name:     "dir/f2",
-		Content:  []byte("blabla"),
-		Branches: []string{"master"},
-	}); err != nil {
-		t.Fatalf("Add: %v", err)
-	}
-
-	s := searcherForTest(t, b)
-	srv := Server{
-		Searcher: s,
-		Top:      Top,
-		HTML:     true,
-		Print:    true,
-	}
-
-	mux, err := NewMux(&srv)
-	if err != nil {
-		t.Fatalf("NewMux: %v", err)
-	}
-
-	ts := httptest.NewServer(mux)
-	defer ts.Close()
-
-	for req, needles := range map[string][]string{
-		"/print?q=bla&r=name&f=f2": {
-			`pre id="l1" class="inline-pre"><span class="noselect"><a href="#l1">`,
-		},
-	} {
-		checkNeedles(t, ts, req, needles)
-	}
-}
-
-func TestPrintDefault(t *testing.T) {
-	b, err := zoekt.NewIndexBuilder(&zoekt.Repository{
-		Name:     "name",
-		URL:      "repo-url",
-		Branches: []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
-	})
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-	if err := b.Add(zoekt.Document{
-		Name:     "f2",
-		Content:  []byte("to carry water in the no later bla"),
-		Branches: []string{"master"},
-	}); err != nil {
-		t.Fatalf("Add: %v", err)
-	}
-	s := searcherForTest(t, b)
-	srv := Server{
-		Searcher: s,
-		Top:      Top,
-		HTML:     true,
-	}
-
-	mux, err := NewMux(&srv)
-	if err != nil {
-		t.Fatalf("NewMux: %v", err)
-	}
-
-	ts := httptest.NewServer(mux)
-	defer ts.Close()
-
-	for req, needles := range map[string][]string{
-		"/search?q=water": {
-			`href="print?`,
-		},
-	} {
-		checkNeedles(t, ts, req, needles)
-	}
-}
-
-func checkNeedles(t *testing.T, ts *httptest.Server, req string, needles []string) {
-	res, err := http.Get(ts.URL + req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	resultBytes, err := ioutil.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	result := string(resultBytes)
-	for _, want := range needles {
-		if !strings.Contains(result, want) {
-			t.Errorf("query %q: result did not have %q: %s", req, want, result)
-		}
-	}
-	if notWant := "crashed"; strings.Contains(result, notWant) {
-		t.Errorf("result has %q: %s", notWant, result)
-	}
-	if notWant := "bytes skipped)..."; strings.Contains(result, notWant) {
-		t.Errorf("result has %q: %s", notWant, result)
-	}
-}
-
-type crashSearcher struct {
-	zoekt.Searcher
-}
-
-func (s *crashSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) {
-	res := zoekt.SearchResult{}
-	res.Stats.Crashes = 1
-	return &res, nil
-}
-
-func TestCrash(t *testing.T) {
-	srv := Server{
-		Searcher: &crashSearcher{},
-		Top:      Top,
-		HTML:     true,
-	}
-
-	mux, err := NewMux(&srv)
-	if err != nil {
-		t.Fatalf("NewMux: %v", err)
-	}
-
-	ts := httptest.NewServer(mux)
-	defer ts.Close()
-
-	res, err := http.Get(ts.URL + "/search?q=water")
-	if err != nil {
-		t.Fatal(err)
-	}
-	resultBytes, err := ioutil.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	result := string(resultBytes)
-	if want := "1 shards crashed"; !strings.Contains(result, want) {
-		t.Errorf("result did not have %q: %s", want, result)
-	}
-}
-
-func TestHostCustomization(t *testing.T) {
-	b, err := zoekt.NewIndexBuilder(&zoekt.Repository{
-		Name: "name",
-	})
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-	if err := b.Add(zoekt.Document{
-		Name:    "file",
-		Content: []byte("bla"),
-	}); err != nil {
-		t.Fatalf("Add: %v", err)
-	}
-
-	s := searcherForTest(t, b)
-	srv := Server{
-		Searcher: s,
-		Top:      Top,
-		HTML:     true,
-		HostCustomQueries: map[string]string{
-			"myproject.io": "r:myproject",
-		},
-	}
-
-	mux, err := NewMux(&srv)
-	if err != nil {
-		t.Fatalf("NewMux: %v", err)
-	}
-
-	ts := httptest.NewServer(mux)
-	defer ts.Close()
-
-	req, err := http.NewRequest("GET", ts.URL, &bytes.Buffer{})
-	if err != nil {
-		t.Fatalf("NewRequest: %v", err)
-	}
-	req.Host = "myproject.io"
-	res, err := (&http.Client{}).Do(req)
-	if err != nil {
-		t.Fatalf("Do(%v): %v", req, err)
-	}
-	resultBytes, err := ioutil.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		t.Fatalf("ReadAll: %v", err)
-	}
-
-	if got, want := string(resultBytes), "r:myproject"; !strings.Contains(got, want) {
-		t.Fatalf("got %s, want substring %q", got, want)
-	}
-}
-
-func TestDupResult(t *testing.T) {
-	b, err := zoekt.NewIndexBuilder(&zoekt.Repository{
-		Name: "name",
-	})
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	for i := 0; i < 2; i++ {
-		if err := b.Add(zoekt.Document{
-			Name:    fmt.Sprintf("file%d", i),
-			Content: []byte("bla"),
-		}); err != nil {
-			t.Fatalf("Add: %v", err)
-		}
-	}
-	s := searcherForTest(t, b)
-	srv := Server{
-		Searcher: s,
-		Top:      Top,
-		HTML:     true,
-	}
-
-	mux, err := NewMux(&srv)
-	if err != nil {
-		t.Fatalf("NewMux: %v", err)
-	}
-
-	ts := httptest.NewServer(mux)
-	defer ts.Close()
-
-	req, err := http.NewRequest("GET", ts.URL+"/search?q=bla", &bytes.Buffer{})
-	if err != nil {
-		t.Fatalf("NewRequest: %v", err)
-	}
-	res, err := (&http.Client{}).Do(req)
-	if err != nil {
-		t.Fatalf("Do(%v): %v", req, err)
-	}
-	resultBytes, err := ioutil.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		t.Fatalf("ReadAll: %v", err)
-	}
-
-	if got, want := string(resultBytes), "Duplicate result"; !strings.Contains(got, want) {
-		t.Fatalf("got %s, want substring %q", got, want)
-	}
-}
-
-func TestTruncateLine(t *testing.T) {
-	b, err := zoekt.NewIndexBuilder(&zoekt.Repository{
-		Name: "name",
-	})
-	if err != nil {
-		t.Fatalf("NewIndexBuilder: %v", err)
-	}
-
-	largePadding := bytes.Repeat([]byte{'a'}, 100*1000) // 100kb
-	if err := b.Add(zoekt.Document{
-		Name:    "file",
-		Content: append(append(largePadding, []byte("helloworld")...), largePadding...),
-	}); err != nil {
-		t.Fatalf("Add: %v", err)
-	}
-	s := searcherForTest(t, b)
-	srv := Server{
-		Searcher: s,
-		Top:      Top,
-		HTML:     true,
-	}
-
-	mux, err := NewMux(&srv)
-	if err != nil {
-		t.Fatalf("NewMux: %v", err)
-	}
-
-	ts := httptest.NewServer(mux)
-	defer ts.Close()
-
-	req, err := http.NewRequest("GET", ts.URL+"/search?q=helloworld", &bytes.Buffer{})
-	if err != nil {
-		t.Fatalf("NewRequest: %v", err)
-	}
-	res, err := (&http.Client{}).Do(req)
-	if err != nil {
-		t.Fatalf("Do(%v): %v", req, err)
-	}
-	resultBytes, err := ioutil.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		t.Fatalf("ReadAll: %v", err)
-	}
-
-	if got, want := len(resultBytes)/1000, 10; got > want {
-		t.Fatalf("got %dkb response, want <= %dkb", got, want)
-	}
-	result := string(resultBytes)
-	if want := "aa<b>helloworld</b>aa"; !strings.Contains(result, want) {
-		t.Fatalf("got %s, want substring %q", result, want)
-	}
-	if want := "bytes skipped)..."; !strings.Contains(result, want) {
-		t.Fatalf("got %s, want substring %q", result, want)
-	}
-}
diff --git a/web/server.go b/web/server.go
deleted file mode 100644
index 94a16ac..0000000
--- a/web/server.go
+++ /dev/null
@@ -1,582 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package web
-
-import (
-	"bytes"
-	"fmt"
-	"html/template"
-	"log"
-	"net"
-	"net/http"
-	"regexp"
-	"regexp/syntax"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"golang.org/x/net/context"
-
-	"github.com/google/zoekt"
-	"github.com/google/zoekt/query"
-)
-
-var Funcmap = template.FuncMap{
-	"Inc": func(orig int) int {
-		return orig + 1
-	},
-	"More": func(orig int) int {
-		return orig * 3
-	},
-	"HumanUnit": func(orig int64) string {
-		b := orig
-		suffix := ""
-		if orig > 10*(1<<30) {
-			suffix = "G"
-			b = orig / (1 << 30)
-		} else if orig > 10*(1<<20) {
-			suffix = "M"
-			b = orig / (1 << 20)
-		} else if orig > 10*(1<<10) {
-			suffix = "K"
-			b = orig / (1 << 10)
-		}
-
-		return fmt.Sprintf("%d%s", b, suffix)
-	},
-	"LimitPre": func(limit int, pre string) string {
-		if len(pre) < limit {
-			return pre
-		}
-		return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
-	},
-	"LimitPost": func(limit int, post string) string {
-		if len(post) < limit {
-			return post
-		}
-		return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
-	},
-}
-
-const defaultNumResults = 50
-
-type Server struct {
-	Searcher zoekt.Searcher
-
-	// Serve HTML interface
-	HTML bool
-
-	// If set, show files from the index.
-	Print bool
-
-	// Version string for this server.
-	Version string
-
-	// Depending on the Host header, add a query to the entry
-	// page. For example, when serving on "search.myproject.org"
-	// we could add "r:myproject" automatically.  This allows a
-	// single instance to serve as search engine for multiple
-	// domains.
-	HostCustomQueries map[string]string
-
-	// This should contain the following templates: "didyoumean"
-	// (for suggestions), "repolist" (for the repo search result
-	// page), "result" for the search results, "search" (for the
-	// opening page), "box" for the search query input element and
-	// "print" for the show file functionality.
-	Top *template.Template
-
-	didYouMean *template.Template
-	repolist   *template.Template
-	search     *template.Template
-	result     *template.Template
-	print      *template.Template
-	about      *template.Template
-	robots     *template.Template
-
-	startTime time.Time
-
-	templateMu    sync.Mutex
-	templateCache map[string]*template.Template
-
-	lastStatsMu sync.Mutex
-	lastStats   *zoekt.RepoStats
-	lastStatsTS time.Time
-}
-
-func (s *Server) getTemplate(str string) *template.Template {
-	s.templateMu.Lock()
-	defer s.templateMu.Unlock()
-	t := s.templateCache[str]
-	if t != nil {
-		return t
-	}
-
-	t, err := template.New("cache").Parse(str)
-	if err != nil {
-		log.Printf("template parse error: %v", err)
-		t = template.Must(template.New("empty").Parse(""))
-	}
-	s.templateCache[str] = t
-	return t
-}
-
-func NewMux(s *Server) (*http.ServeMux, error) {
-	s.print = s.Top.Lookup("print")
-	if s.print == nil {
-		return nil, fmt.Errorf("missing template 'print'")
-	}
-
-	for k, v := range map[string]**template.Template{
-		"didyoumean": &s.didYouMean,
-		"results":    &s.result,
-		"print":      &s.print,
-		"search":     &s.search,
-		"repolist":   &s.repolist,
-		"about":      &s.about,
-		"robots":     &s.robots,
-	} {
-		*v = s.Top.Lookup(k)
-		if *v == nil {
-			return nil, fmt.Errorf("missing template %q", k)
-		}
-	}
-
-	s.templateCache = map[string]*template.Template{}
-	s.startTime = time.Now()
-
-	mux := http.NewServeMux()
-
-	if s.HTML {
-		mux.HandleFunc("/robots.txt", s.serveRobots)
-		mux.HandleFunc("/search", s.serveSearch)
-		mux.HandleFunc("/", s.serveSearchBox)
-		mux.HandleFunc("/about", s.serveAbout)
-		mux.HandleFunc("/print", s.servePrint)
-	}
-
-	return mux, nil
-}
-
-func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
-	err := s.serveSearchErr(w, r)
-
-	if suggest, ok := err.(*query.SuggestQueryError); ok {
-		var buf bytes.Buffer
-		if err := s.didYouMean.Execute(&buf, suggest); err != nil {
-			http.Error(w, err.Error(), http.StatusTeapot)
-		}
-
-		w.Write(buf.Bytes())
-		return
-	}
-
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusTeapot)
-	}
-}
-
-func (s *Server) serveSearchErr(w http.ResponseWriter, r *http.Request) error {
-	qvals := r.URL.Query()
-	queryStr := qvals.Get("q")
-	if queryStr == "" {
-		return fmt.Errorf("no query found")
-	}
-
-	q, err := query.Parse(queryStr)
-	if err != nil {
-		return err
-	}
-
-	repoOnly := true
-	query.VisitAtoms(q, func(q query.Q) {
-		_, ok := q.(*query.Repo)
-		repoOnly = repoOnly && ok
-	})
-	if repoOnly {
-		return s.serveListReposErr(q, queryStr, w, r)
-	}
-
-	numStr := qvals.Get("num")
-
-	num, err := strconv.Atoi(numStr)
-	if err != nil || num <= 0 {
-		num = defaultNumResults
-	}
-
-	sOpts := zoekt.SearchOptions{
-		MaxWallTime: 10 * time.Second,
-	}
-
-	sOpts.SetDefaults()
-
-	ctx := r.Context()
-	if result, err := s.Searcher.Search(ctx, q, &zoekt.SearchOptions{EstimateDocCount: true}); err != nil {
-		return err
-	} else if numdocs := result.ShardFilesConsidered; numdocs > 10000 {
-		// If the search touches many shards and many files, we
-		// have to limit the number of matches.  This setting
-		// is based on the number of documents eligible after
-		// considering reponames, so large repos (both
-		// android, chromium are about 500k files) aren't
-		// covered fairly.
-
-		// 10k docs, 50 num -> max match = (250 + 250 / 10)
-		sOpts.ShardMaxMatchCount = num*5 + (5*num)/(numdocs/1000)
-
-		// 10k docs, 50 num -> max important match = 4
-		sOpts.ShardMaxImportantMatch = num/20 + num/(numdocs/500)
-	} else {
-		// Virtually no limits for a small corpus; important
-		// matches are just as expensive as normal matches.
-		n := numdocs + num*100
-		sOpts.ShardMaxImportantMatch = n
-		sOpts.ShardMaxMatchCount = n
-		sOpts.TotalMaxMatchCount = n
-		sOpts.TotalMaxImportantMatch = n
-	}
-	sOpts.MaxDocDisplayCount = num
-
-	result, err := s.Searcher.Search(ctx, q, &sOpts)
-	if err != nil {
-		return err
-	}
-
-	fileMatches, err := s.formatResults(result, queryStr, s.Print)
-	if err != nil {
-		return err
-	}
-
-	res := ResultInput{
-		Last: LastInput{
-			Query:     queryStr,
-			Num:       num,
-			AutoFocus: true,
-		},
-		Stats:         result.Stats,
-		Query:         q.String(),
-		QueryStr:      queryStr,
-		SearchOptions: sOpts.String(),
-		FileMatches:   fileMatches,
-	}
-	if res.Stats.Wait < res.Stats.Duration/10 {
-		// Suppress queueing stats if they are neglible.
-		res.Stats.Wait = 0
-	}
-
-	var buf bytes.Buffer
-	if err := s.result.Execute(&buf, &res); err != nil {
-		return err
-	}
-
-	w.Write(buf.Bytes())
-	return nil
-}
-
-func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
-	err := s.servePrintErr(w, r)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusTeapot)
-	}
-}
-
-const statsStaleNess = 30 * time.Second
-
-func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
-	s.lastStatsMu.Lock()
-	stats := s.lastStats
-	if time.Since(s.lastStatsTS) > statsStaleNess {
-		stats = nil
-	}
-	s.lastStatsMu.Unlock()
-
-	if stats != nil {
-		return stats, nil
-	}
-
-	repos, err := s.Searcher.List(ctx, &query.Const{Value: true})
-	if err != nil {
-		return nil, err
-	}
-
-	stats = &zoekt.RepoStats{}
-	names := map[string]struct{}{}
-	for _, r := range repos.Repos {
-		stats.Add(&r.Stats)
-		names[r.Repository.Name] = struct{}{}
-	}
-	stats.Repos = len(names)
-
-	s.lastStatsMu.Lock()
-	s.lastStatsTS = time.Now()
-	s.lastStats = stats
-	s.lastStatsMu.Unlock()
-
-	return stats, nil
-}
-
-func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
-	stats, err := s.fetchStats(r.Context())
-	if err != nil {
-		return err
-	}
-	d := SearchBoxInput{
-		Last: LastInput{
-			Num:       defaultNumResults,
-			AutoFocus: true,
-		},
-		Stats:   stats,
-		Version: s.Version,
-		Uptime:  time.Since(s.startTime),
-	}
-
-	d.Last.Query = r.URL.Query().Get("q")
-	if d.Last.Query == "" {
-		custom := s.HostCustomQueries[r.Host]
-		if custom == "" {
-			host, _, _ := net.SplitHostPort(r.Host)
-			custom = s.HostCustomQueries[host]
-		}
-
-		if custom != "" {
-			d.Last.Query = custom + " "
-		}
-	}
-
-	var buf bytes.Buffer
-	if err := s.search.Execute(&buf, &d); err != nil {
-		return err
-	}
-	w.Write(buf.Bytes())
-	return nil
-}
-
-func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
-	if err := s.serveSearchBoxErr(w, r); err != nil {
-		http.Error(w, err.Error(), http.StatusTeapot)
-	}
-}
-
-func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
-	stats, err := s.fetchStats(r.Context())
-	if err != nil {
-		return err
-	}
-
-	d := SearchBoxInput{
-		Stats:   stats,
-		Version: s.Version,
-		Uptime:  time.Since(s.startTime),
-	}
-
-	var buf bytes.Buffer
-	if err := s.about.Execute(&buf, &d); err != nil {
-		return err
-	}
-	w.Write(buf.Bytes())
-	return nil
-}
-
-func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
-	if err := s.serveAboutErr(w, r); err != nil {
-		http.Error(w, err.Error(), http.StatusTeapot)
-	}
-}
-
-func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
-	data := struct{}{}
-	var buf bytes.Buffer
-	if err := s.robots.Execute(&buf, &data); err != nil {
-		return err
-	}
-	w.Write(buf.Bytes())
-	return nil
-}
-
-func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
-	if err := s.serveRobotsErr(w, r); err != nil {
-		http.Error(w, err.Error(), http.StatusTeapot)
-	}
-}
-
-func (s *Server) serveListReposErr(q query.Q, qStr string, w http.ResponseWriter, r *http.Request) error {
-	ctx := r.Context()
-	repos, err := s.Searcher.List(ctx, q)
-	if err != nil {
-		return err
-	}
-
-	qvals := r.URL.Query()
-	order := qvals.Get("order")
-	switch order {
-	case "", "name", "revname":
-		sort.Slice(repos.Repos, func(i, j int) bool {
-			return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
-		})
-	case "size", "revsize":
-		sort.Slice(repos.Repos, func(i, j int) bool {
-			return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
-		})
-	case "time", "revtime":
-		sort.Slice(repos.Repos, func(i, j int) bool {
-			return repos.Repos[i].IndexMetadata.IndexTime.Before(
-				repos.Repos[j].IndexMetadata.IndexTime)
-		})
-	default:
-		return fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
-	}
-	if strings.HasPrefix(order, "rev") {
-		for i, j := 0, len(repos.Repos)-1; i < j; {
-			repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
-			i++
-			j--
-
-		}
-	}
-
-	aggregate := zoekt.RepoStats{
-		Repos: len(repos.Repos),
-	}
-	for _, s := range repos.Repos {
-		aggregate.Add(&s.Stats)
-	}
-	res := RepoListInput{
-		Last: LastInput{
-			Query:     qStr,
-			AutoFocus: true,
-		},
-		Stats: aggregate,
-	}
-
-	numStr := qvals.Get("num")
-	num, err := strconv.Atoi(numStr)
-	if err != nil || num <= 0 {
-		num = 0
-	}
-	if num > 0 {
-		if num > len(repos.Repos) {
-			num = len(repos.Repos)
-		}
-
-		repos.Repos = repos.Repos[:num]
-	}
-
-	for _, r := range repos.Repos {
-		t := s.getTemplate(r.Repository.CommitURLTemplate)
-
-		repo := Repository{
-			Name:      r.Repository.Name,
-			URL:       r.Repository.URL,
-			IndexTime: r.IndexMetadata.IndexTime,
-			Size:      r.Stats.ContentBytes,
-			Files:     int64(r.Stats.Documents),
-		}
-		for _, b := range r.Repository.Branches {
-			var buf bytes.Buffer
-			if err := t.Execute(&buf, b); err != nil {
-				return err
-			}
-			repo.Branches = append(repo.Branches,
-				Branch{
-					Name:    b.Name,
-					Version: b.Version,
-					URL:     buf.String(),
-				})
-		}
-		res.Repos = append(res.Repos, repo)
-	}
-
-	var buf bytes.Buffer
-	if err := s.repolist.Execute(&buf, &res); err != nil {
-		return err
-	}
-
-	w.Write(buf.Bytes())
-	return nil
-}
-
-func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
-	qvals := r.URL.Query()
-	fileStr := qvals.Get("f")
-	repoStr := qvals.Get("r")
-	queryStr := qvals.Get("q")
-	numStr := qvals.Get("num")
-	num, err := strconv.Atoi(numStr)
-	if err != nil || num <= 0 {
-		num = defaultNumResults
-	}
-
-	re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
-	if err != nil {
-		return err
-	}
-	qs := []query.Q{
-		&query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
-		&query.Repo{Pattern: repoStr},
-	}
-
-	if branchStr := qvals.Get("b"); branchStr != "" {
-		qs = append(qs, &query.Branch{Pattern: branchStr})
-	}
-
-	q := &query.And{Children: qs}
-
-	sOpts := zoekt.SearchOptions{
-		Whole: true,
-	}
-
-	ctx := r.Context()
-	result, err := s.Searcher.Search(ctx, q, &sOpts)
-	if err != nil {
-		return err
-	}
-
-	if len(result.Files) != 1 {
-		var ss []string
-		for _, n := range result.Files {
-			ss = append(ss, n.FileName)
-		}
-		return fmt.Errorf("ambiguous result: %v", ss)
-	}
-
-	f := result.Files[0]
-
-	byteLines := bytes.Split(f.Content, []byte{'\n'})
-	strLines := make([]string, 0, len(byteLines))
-	for _, l := range byteLines {
-		strLines = append(strLines, string(l))
-	}
-
-	d := PrintInput{
-		Name:  f.FileName,
-		Repo:  f.Repository,
-		Lines: strLines,
-		Last: LastInput{
-			Query:     queryStr,
-			Num:       num,
-			AutoFocus: false,
-		},
-	}
-
-	var buf bytes.Buffer
-	if err := s.print.Execute(&buf, &d); err != nil {
-		return err
-	}
-
-	w.Write(buf.Bytes())
-	return nil
-}
diff --git a/web/snippets.go b/web/snippets.go
deleted file mode 100644
index 1b6112e..0000000
--- a/web/snippets.go
+++ /dev/null
@@ -1,149 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package web
-
-import (
-	"bytes"
-	"html/template"
-	"log"
-	"net/url"
-	"strconv"
-	"strings"
-
-	"github.com/google/zoekt"
-)
-
-func (s *Server) formatResults(result *zoekt.SearchResult, query string, localPrint bool) ([]*FileMatch, error) {
-	var fmatches []*FileMatch
-
-	templateMap := map[string]*template.Template{}
-	fragmentMap := map[string]*template.Template{}
-	if !localPrint {
-		for repo, str := range result.RepoURLs {
-			if str != "" {
-				templateMap[repo] = s.getTemplate(str)
-			}
-		}
-		for repo, str := range result.LineFragments {
-			if str != "" {
-				fragmentMap[repo] = s.getTemplate(str)
-			}
-		}
-	}
-	getFragment := func(repo string, linenum int) string {
-		tpl := fragmentMap[repo]
-
-		if tpl == nil || localPrint {
-			return "#l" + strconv.Itoa(linenum)
-		}
-
-		var buf bytes.Buffer
-		if err := tpl.Execute(&buf, map[string]string{
-			"LineNumber": strconv.Itoa(linenum),
-		}); err != nil {
-			log.Printf("fragment template: %v", err)
-			return ""
-		}
-		return buf.String()
-	}
-	getURL := func(repo, filename string, branches []string, version string) string {
-		tpl := templateMap[repo]
-		if localPrint || tpl == nil {
-			v := make(url.Values)
-			v.Add("r", repo)
-			v.Add("f", filename)
-			v.Add("q", query)
-			if len(branches) > 0 {
-				v.Add("b", branches[0])
-			}
-			return "print?" + v.Encode()
-		}
-
-		var buf bytes.Buffer
-		b := ""
-		if len(branches) > 0 {
-			b = branches[0]
-		}
-		err := tpl.Execute(&buf, map[string]string{
-			"Branch":  b,
-			"Version": version,
-			"Path":    filename,
-		})
-		if err != nil {
-			log.Printf("url template: %v", err)
-			return ""
-		}
-		return buf.String()
-	}
-
-	// hash => result-id
-	seenFiles := map[string]string{}
-	for _, f := range result.Files {
-		fMatch := FileMatch{
-			FileName: f.FileName,
-			Repo:     f.Repository,
-			ResultID: f.Repository + ":" + f.FileName,
-			Branches: f.Branches,
-			Language: f.Language,
-		}
-
-		if dup, ok := seenFiles[string(f.Checksum)]; ok {
-			fMatch.DuplicateID = dup
-		} else {
-			seenFiles[string(f.Checksum)] = fMatch.ResultID
-		}
-
-		if f.SubRepositoryName != "" {
-			fn := strings.TrimPrefix(fMatch.FileName[len(f.SubRepositoryPath):], "/")
-			fMatch.URL = getURL(f.SubRepositoryName, fn, f.Branches, f.Version)
-		} else {
-			fMatch.URL = getURL(f.Repository, f.FileName, f.Branches, f.Version)
-		}
-
-		for _, m := range f.LineMatches {
-			fragment := getFragment(f.Repository, m.LineNumber)
-			if !strings.HasPrefix(fragment, "#") && !strings.HasPrefix(fragment, ";") {
-				// TODO - remove this is backward compatibility glue.
-				fragment = "#" + fragment
-			}
-			md := Match{
-				FileName: f.FileName,
-				LineNum:  m.LineNumber,
-				URL:      fMatch.URL + fragment,
-			}
-
-			lastEnd := 0
-			line := m.Line
-			for i, f := range m.LineFragments {
-				l := f.LineOffset
-				e := l + f.MatchLength
-
-				frag := Fragment{
-					Pre:   string(line[lastEnd:l]),
-					Match: string(line[l:e]),
-				}
-				if i == len(m.LineFragments)-1 {
-					frag.Post = string(m.Line[e:])
-				}
-
-				md.Fragments = append(md.Fragments, frag)
-				lastEnd = e
-			}
-			fMatch.Matches = append(fMatch.Matches, md)
-		}
-		fmatches = append(fmatches, &fMatch)
-	}
-	return fmatches, nil
-}
diff --git a/web/templates.go b/web/templates.go
deleted file mode 100644
index eea9bb7..0000000
--- a/web/templates.go
+++ /dev/null
@@ -1,409 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package web
-
-import (
-	"html/template"
-	"log"
-)
-
-// Top provides the standard templates in parsed form
-var Top = template.New("top").Funcs(Funcmap)
-
-// TemplateText contains the text of the standard templates.
-var TemplateText = map[string]string{
-
-	"didyoumean": `
-<html>
-<head>
-  <title>Error</title>
-</head>
-<body>
-  <p>{{.Message}}. Did you mean <a href="/search?q={{.Suggestion}}">{{.Suggestion}}</a> ?
-</body>
-</html>
-`,
-
-	"head": `
-<head>
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<!-- Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) -->
-<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
-<style>
-  #navsearchbox { width: 350px !important; }
-  #maxhits { width: 100px !important; }
-  .label-dup {
-    border-width: 1px !important;
-    border-style: solid !important;
-    border-color: #aaa !important;
-    color: black;
-  }
-  .noselect {
-    user-select: none;
-  }
-  a.label-dup:hover {
-    color: black;
-    background: #ddd;
-  }
-  .result {
-    display: block;
-    content: " ";
-    visibility: hidden;
-  }
-  .container-results {
-     overflow: auto;
-     max-height: calc(100% - 72px);
-  }
-  .inline-pre {
-     border: unset;
-     background-color: unset;
-     margin: unset;
-     padding: unset;
-     overflow: unset;
-  }
-  :target { background-color: #ccf; }
-  table tbody tr td { border: none !important; padding: 2px !important; }
-</style>
-</head>
-  `,
-
-	"jsdep": `
-<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
-<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
-`,
-
-	// the template for the search box.
-	"searchbox": `
-<form action="search">
-  <div class="form-group form-group-lg">
-    <div class="input-group input-group-lg">
-      <input class="form-control" placeholder="Search for some code..." autofocus
-              {{if .Query}}
-              value={{.Query}}
-              {{end}}
-              id="searchbox" type="text" name="q">
-      <div class="input-group-btn">
-        <button class="btn btn-primary">Search</button>
-      </div>
-    </div>
-  </div>
-</form>
-`,
-
-	"navbar": `
-<nav class="navbar navbar-default">
-  <div class="container-fluid">
-    <div class="navbar-header">
-      <a class="navbar-brand" href="/">Zoekt</a>
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar-collapse" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-    </div>
-    <div class="navbar-collapse collapse" id="navbar-collapse" aria-expanded="false" style="height: 1px;">
-      <form class="navbar-form navbar-left" action="search">
-        <div class="form-group">
-          <input class="form-control"
-                placeholder="Search for some code..." role="search"
-                id="navsearchbox" type="text" name="q" autofocus
-                {{if .Query}}
-                value={{.Query}}
-                {{end}}>
-          <div class="input-group">
-            <div class="input-group-addon">Max Results</div>
-            <input class="form-control" type="number" id="maxhits" name="num" value="{{.Num}}">
-          </div>
-          <button class="btn btn-primary">Search</button>
-        </div>
-      </form>
-    </div>
-  </div>
-</nav>
-<script>
-document.onkeydown=function(e){
-  var e = e || window.event;
-  if (e.key == "/") {
-    var navbox = document.getElementById("navsearchbox");
-    if (document.activeElement !== navbox) {
-      navbox.focus();
-      return false;
-    }
-  }
-};
-</script>
-`,
-	// search box for the entry page.
-	"search": `
-<html>
-{{template "head"}}
-<title>Zoekt, en gij zult spinazie eten</title>
-<body>
-  <div class="jumbotron">
-    <div class="container">
-      {{template "searchbox" .Last}}
-    </div>
-  </div>
-
-  <div class="container">
-    <div class="row">
-      <div class="col-md-8">
-        <h3>Search examples:</h3>
-        <dl class="dl-horizontal">
-          <dt><a href="search?q=needle">needle</a></dt><dd>search for "needle"</dd>
-          <dt><a href="search?q=thread+or+needle">thread or needle</a></dt><dd>search for either "thread" or "needle"</dd>
-          <dt><a href="search?q=class+needle">class needle</a></span></dt><dd>search for files containing both "class" and "needle"</dd>
-          <dt><a href="search?q=class+Needle">class Needle</a></dt><dd>search for files containing both "class" (case insensitive) and "Needle" (case sensitive)</dd>
-          <dt><a href="search?q=class+Needle+case:yes">class Needle case:yes</a></dt><dd>search for files containing "class" and "Needle", both case sensitively</dd>
-          <dt><a href="search?q=%22class Needle%22">"class Needle"</a></dt><dd>search for files with the phrase "class Needle"</dd>
-          <dt><a href="search?q=needle+-hay">needle -hay</a></dt><dd>search for files with the word "needle" but not the word "hay"</dd>
-          <dt><a href="search?q=path+file:java">path file:java</a></dt><dd>search for the word "path" in files whose name contains "java"</dd>
-          <dt><a href="search?q=needle+lang%3Apython&num=50">needle lang:python</a></dt><dd>search for "needle" in Python source code</dd>
-          <dt><a href="search?q=f:%5C.c%24">f:\.c$</a></dt><dd>search for files whose name ends with ".c"</dd>
-          <dt><a href="search?q=path+-file:java">path -file:java</a></dt><dd>search for the word "path" excluding files whose name contains "java"</dd>
-          <dt><a href="search?q=foo.*bar">foo.*bar</a></dt><dd>search for the regular expression "foo.*bar"</dd>
-          <dt><a href="search?q=-%28Path File%29 Stream">-(Path File) Stream</a></dt><dd>search "Stream", but exclude files containing both "Path" and "File"</dd>
-          <dt><a href="search?q=-Path%5c+file+Stream">-Path\ file Stream</a></dt><dd>search "Stream", but exclude files containing "Path File"</dd>
-          <dt><a href="search?q=sym:data">sym:data</a></span></dt><dd>search for symbol definitions containing "data"</dd>
-          <dt><a href="search?q=phone+r:droid">phone r:droid</a></dt><dd>search for "phone" in repositories whose name contains "droid"</dd>
-          <dt><a href="search?q=phone+b:master">phone b:master</a></dt><dd>for Git repos, find "phone" in files in branches whose name contains "master".</dd>
-          <dt><a href="search?q=phone+b:HEAD">phone b:HEAD</a></dt><dd>for Git repos, find "phone" in the default ('HEAD') branch.</dd>
-        </dl>
-      </div>
-      <div class="col-md-4">
-        <h3>To list repositories, try:</h3>
-        <dl class="dl-horizontal">
-          <dt><a href="search?q=r:droid">r:droid</a></dt><dd>list repositories whose name contains "droid".</dd>
-          <dt><a href="search?q=r:go+-r:google">r:go -r:google</a></dt><dd>list repositories whose name contains "go" but not "google".</dd>
-        </dl>
-      </div>
-    </div>
-  </div>
-  <nav class="navbar navbar-default navbar-bottom">
-    <div class="container">
-      {{template "footerBoilerplate"}}
-      <p class="navbar-text navbar-right">
-        Used {{HumanUnit .Stats.IndexBytes}} mem for
-        {{.Stats.Documents}} documents ({{HumanUnit .Stats.ContentBytes}})
-        from {{.Stats.Repos}} repositories.
-      </p>
-    </div>
-  </nav>
-</body>
-</html>
-`,
-	"footerBoilerplate": `<a class="navbar-text" href="about">About</a>`,
-	"results": `
-<html>
-{{template "head"}}
-<title>Results for {{.QueryStr}}</title>
-<script>
-  function zoektAddQ(atom) {
-      window.location.href = "/search?q=" + escape("{{.QueryStr}}" + " " + atom) +
-	  "&" + "num=" + {{.Last.Num}};
-  }
-</script>
-<body id="results">
-  {{template "navbar" .Last}}
-  <div class="container-fluid container-results">
-    <h5>
-      {{if .Stats.Crashes}}<br><b>{{.Stats.Crashes}} shards crashed</b><br>{{end}}
-      {{ $fileCount := len .FileMatches }}
-      Found {{.Stats.MatchCount}} results in {{.Stats.FileCount}} files{{if or (lt $fileCount .Stats.FileCount) (or (gt .Stats.ShardsSkipped 0) (gt .Stats.FilesSkipped 0)) }},
-        showing top {{ $fileCount }} files (<a rel="nofollow"
-           href="search?q={{.Last.Query}}&num={{More .Last.Num}}">show more</a>).
-      {{else}}.{{end}}
-    </h5>
-    {{range .FileMatches}}
-    <table class="table table-hover table-condensed">
-      <thead>
-        <tr>
-          <th>
-            {{if .URL}}<a name="{{.ResultID}}" class="result"></a><a href="{{.URL}}" >{{else}}<a name="{{.ResultID}}">{{end}}
-            <small>
-              {{.Repo}}:{{.FileName}}</a>:
-              <span style="font-weight: normal">[ {{if .Branches}}{{range .Branches}}<span class="label label-default">{{.}}</span>,{{end}}{{end}} ]</span>
-              {{if .Language}}<button
-                   title="restrict search to files written in {{.Language}}"
-                   onclick="zoektAddQ('lang:{{.Language}}')" class="label label-primary">language {{.Language}}</button></span>{{end}}
-              {{if .DuplicateID}}<a class="label label-dup" href="#{{.DuplicateID}}">Duplicate result</a>{{end}}
-            </small>
-          </th>
-        </tr>
-      </thead>
-      {{if not .DuplicateID}}
-      <tbody>
-        {{range .Matches}}
-        <tr>
-          <td style="background-color: rgba(238, 238, 255, 0.6);">
-            <pre class="inline-pre"><span class="noselect">{{if .URL}}<a href="{{.URL}}">{{end}}<u>{{.LineNum}}</u>{{if .URL}}</a>{{end}}: </span>{{range .Fragments}}{{LimitPre 100 .Pre}}<b>{{.Match}}</b>{{LimitPost 100 .Post}}{{end}}</pre>
-          </td>
-        </tr>
-        {{end}}
-      </tbody>
-      {{end}}
-    </table>
-    {{end}}
-
-  <nav class="navbar navbar-default navbar-bottom">
-    <div class="container">
-      {{template "footerBoilerplate"}}
-      <p class="navbar-text navbar-right">
-      Took {{.Stats.Duration}}{{if .Stats.Wait}}(queued: {{.Stats.Wait}}){{end}} for
-      {{HumanUnit .Stats.IndexBytesLoaded}}B index data,
-      {{.Stats.NgramMatches}} ngram matches,
-      {{.Stats.FilesConsidered}} docs considered,
-      {{.Stats.FilesLoaded}} docs ({{HumanUnit .Stats.ContentBytesLoaded}}B)
-      loaded{{if or .Stats.FilesSkipped .Stats.ShardsSkipped}},
-      {{.Stats.FilesSkipped}} docs and {{.Stats.ShardsSkipped}} shards skipped{{else}}.{{end}}
-      </p>
-    </div>
-  </nav>
-  </div>
-  {{ template "jsdep"}}
-</body>
-</html>
-`,
-
-	"repolist": `
-<html>
-{{template "head"}}
-<body id="results">
-  <div class="container">
-    {{template "navbar" .Last}}
-    <div><b>
-    Found {{.Stats.Repos}} repositories ({{.Stats.Documents}} files, {{HumanUnit .Stats.ContentBytes}}b content)
-    </b></div>
-    <table class="table table-hover table-condensed">
-      <thead>
-	<tr>
-	  <th>Name <a href="/search?q={{.Last.Query}}&order=name">▼</a><a href="/search?q={{.Last.Query}}&order=revname">▲</a></th>
-	  <th>Last updated <a href="/search?q={{.Last.Query}}&order=revtime">▼</a><a href="/search?q={{.Last.Query}}&order=time">▲</a></th>
-	  <th>Branches</th>
-	  <th>Size <a href="/search?q={{.Last.Query}}&order=revsize">▼</a><a href="/search?q={{.Last.Query}}&order=size">▲</a></th>
-	</tr>
-      </thead>
-      <tbody>
-	{{range .Repos}}
-	<tr>
-	  <td>{{if .URL}}<a href="{{.URL}}">{{end}}{{.Name}}{{if .URL}}</a>{{end}}</td>
-	  <td><small>{{.IndexTime.Format "Jan 02, 2006 15:04"}}</small></td>
-	  <td style="vertical-align: middle;">
-	    {{range .Branches}}
-	    {{if .URL}}<tt><a class="label label-default small" href="{{.URL}}">{{end}}{{.Name}}{{if .URL}}</a> </tt>{{end}}&nbsp;
-	    {{end}}
-	  </td>
-	  <td><small>{{HumanUnit .Files}} files ({{HumanUnit .Size}})</small></td>
-	</tr>
-	{{end}}
-      </tbody>
-    </table>
-  </div>
-
-  <nav class="navbar navbar-default navbar-bottom">
-    <div class="container">
-      {{template "footerBoilerplate"}}
-      <p class="navbar-text navbar-right">
-      </p>
-    </div>
-  </nav>
-
-  {{ template "jsdep"}}
-</body>
-</html>
-`,
-
-	"print": `
-<html>
-  {{template "head"}}
-  <title>{{.Repo}}:{{.Name}}</title>
-<body id="results">
-  {{template "navbar" .Last}}
-  <div class="container-fluid container-results" >
-     <div><b>{{.Name}}</b></div>
-     <div class="table table-hover table-condensed" style="overflow:auto; background: #eef;">
-       {{ range $index, $ln := .Lines}}
-	 <pre id="l{{Inc $index}}" class="inline-pre"><span class="noselect"><a href="#l{{Inc $index}}">{{Inc $index}}</a>: </span>{{$ln}}</pre>
-       {{end}}
-     </div>
-  <nav class="navbar navbar-default navbar-bottom">
-    <div class="container">
-      {{template "footerBoilerplate"}}
-      <p class="navbar-text navbar-right">
-      </p>
-    </div>
-  </nav>
-  </div>
- {{ template "jsdep"}}
-</body>
-</html>
-`,
-
-	"about": `
-
-<html>
-  {{template "head"}}
-  <title>About <em>zoekt</em></title>
-<body>
-
-
-  <div class="jumbotron">
-    <div class="container">
-      {{template "searchbox" .Last}}
-    </div>
-  </div>
-
-  <div class="container">
-    <p>
-      This is <a href="http://github.com/google/zoekt"><em>zoekt</em> (IPA: /zukt/)</a>,
-      an open-source full text search engine. It's pronounced roughly as you would
-      pronounce "zooked" in English.
-    </p>
-    <p>
-    {{if .Version}}<em>Zoekt</em> version {{.Version}}, uptime{{else}}Uptime{{end}} {{.Uptime}}
-    </p>
-
-    <p>
-    Used {{HumanUnit .Stats.IndexBytes}} memory for
-    {{.Stats.Documents}} documents ({{HumanUnit .Stats.ContentBytes}})
-    from {{.Stats.Repos}} repositories.
-    </p>
-  </div>
-
-  <nav class="navbar navbar-default navbar-bottom">
-    <div class="container">
-      {{template "footerBoilerplate"}}
-      <p class="navbar-text navbar-right">
-      </p>
-    </div>
-  </nav>
-`,
-	"robots": `
-user-agent: *
-disallow: /search
-`,
-}
-
-func init() {
-	for k, v := range TemplateText {
-		_, err := Top.New(k).Parse(v)
-		if err != nil {
-			log.Panicf("parse(%s): %v:", k, err)
-		}
-	}
-}
diff --git a/write.go b/write.go
deleted file mode 100644
index 7a89167..0000000
--- a/write.go
+++ /dev/null
@@ -1,178 +0,0 @@
-// Copyright 2016 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package zoekt
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/binary"
-	"encoding/json"
-	"io"
-	"sort"
-	"time"
-)
-
-func (w *writer) writeTOC(toc *indexTOC) {
-	// Tagged sections are indicated with a 0 section count.
-	// Tagged sections allow easier forwards and backwards
-	// compatibility when evolving zoekt index files with new
-	// sections.
-	//
-	// A tagged section is:
-	// Varint TagLen, Tag String, Varint SecType, Section
-	//
-	// Section type is indicated because simpleSections and
-	// compoundSections have different lengths.
-	w.U32(0)
-	secs := toc.sectionsTaggedList()
-	for _, s := range secs {
-		w.String(s.tag)
-		w.Varint(uint32(s.sec.kind()))
-		s.sec.write(w)
-	}
-}
-
-func (s *compoundSection) writeStrings(w *writer, strs []*searchableString) {
-	s.start(w)
-	for _, f := range strs {
-		s.addItem(w, f.data)
-	}
-	s.end(w)
-}
-
-func writePostings(w *writer, s *postingsBuilder, ngramText *simpleSection,
-	charOffsets *simpleSection, postings *compoundSection, endRunes *simpleSection) {
-	keys := make(ngramSlice, 0, len(s.postings))
-	for k := range s.postings {
-		keys = append(keys, k)
-	}
-	sort.Sort(keys)
-
-	ngramText.start(w)
-	for _, k := range keys {
-		var buf [8]byte
-		binary.BigEndian.PutUint64(buf[:], uint64(k))
-		w.Write(buf[:])
-	}
-	ngramText.end(w)
-
-	postings.start(w)
-	for _, k := range keys {
-		postings.addItem(w, s.postings[k])
-	}
-	postings.end(w)
-
-	charOffsets.start(w)
-	w.Write(toSizedDeltas(s.runeOffsets))
-	charOffsets.end(w)
-
-	endRunes.start(w)
-	w.Write(toSizedDeltas(s.endRunes))
-	endRunes.end(w)
-}
-
-func (b *IndexBuilder) Write(out io.Writer) error {
-	buffered := bufio.NewWriterSize(out, 1<<20)
-	defer buffered.Flush()
-
-	w := &writer{w: buffered}
-	toc := indexTOC{}
-
-	toc.fileContents.writeStrings(w, b.contentStrings)
-	toc.newlines.start(w)
-	for _, f := range b.contentStrings {
-		toc.newlines.addItem(w, toSizedDeltas(newLinesIndices(f.data)))
-	}
-	toc.newlines.end(w)
-
-	toc.branchMasks.start(w)
-	for _, m := range b.branchMasks {
-		w.U64(m)
-	}
-	toc.branchMasks.end(w)
-
-	toc.fileSections.start(w)
-	for _, s := range b.docSections {
-		toc.fileSections.addItem(w, marshalDocSections(s))
-	}
-	toc.fileSections.end(w)
-
-	writePostings(w, b.contentPostings, &toc.ngramText, &toc.runeOffsets, &toc.postings, &toc.fileEndRunes)
-
-	// names.
-	toc.fileNames.writeStrings(w, b.nameStrings)
-
-	writePostings(w, b.namePostings, &toc.nameNgramText, &toc.nameRuneOffsets, &toc.namePostings, &toc.nameEndRunes)
-
-	toc.subRepos.start(w)
-	w.Write(toSizedDeltas(b.subRepos))
-	toc.subRepos.end(w)
-
-	toc.contentChecksums.start(w)
-	w.Write(b.checksums)
-	toc.contentChecksums.end(w)
-
-	toc.languages.start(w)
-	w.Write(b.languages)
-	toc.languages.end(w)
-
-	toc.runeDocSections.start(w)
-	w.Write(marshalDocSections(b.runeDocSections))
-	toc.runeDocSections.end(w)
-
-	if err := b.writeJSON(&IndexMetadata{
-		IndexFormatVersion:    IndexFormatVersion,
-		IndexTime:             time.Now(),
-		IndexFeatureVersion:   FeatureVersion,
-		IndexMinReaderVersion: WriteMinFeatureVersion,
-		PlainASCII:            b.contentPostings.isPlainASCII && b.namePostings.isPlainASCII,
-		LanguageMap:           b.languageMap,
-		ZoektVersion:          Version,
-	}, &toc.metaData, w); err != nil {
-		return err
-	}
-	if err := b.writeJSON(b.repo, &toc.repoMetaData, w); err != nil {
-		return err
-	}
-
-	var tocSection simpleSection
-
-	tocSection.start(w)
-	w.writeTOC(&toc)
-	tocSection.end(w)
-	tocSection.write(w)
-	return w.err
-}
-
-func (b *IndexBuilder) writeJSON(data interface{}, sec *simpleSection, w *writer) error {
-	blob, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-	sec.start(w)
-	w.Write(blob)
-	sec.end(w)
-	return nil
-}
-
-func newLinesIndices(in []byte) []uint32 {
-	out := make([]uint32, 0, bytes.Count(in, []byte{'\n'}))
-	for i, c := range in {
-		if c == '\n' {
-			out = append(out, uint32(i))
-		}
-	}
-	return out
-}