index: read in ngrams in own function

Our heap profiles are dominated by readIndexData. By moving some of the
work into its own functions, the heap profiles will tell us where in
particular data is allocated from. We suspect most data is from
readNgrams, but want to confirm emperically.

Co-authored-by: Stefan Hengl <stefan@sourcegraph.com>
Change-Id: I85cb05f27ac2c2d9135c816173a8a97cea6a706b

read.go

diff --git a/read.go b/read.go
index f8d07ab..ec54223 100644
--- a/read.go
+++ b/read.go
@@ -183,21 +183,10 @@
 		return nil, err
 	}
 
-	textContent, err := d.readSectionBlob(toc.ngramText)
+	d.ngrams, err = d.readNgrams(toc)
 	if err != nil {
 		return nil, err
 	}
-	postingsIndex := toc.postings.relativeIndex()
-
-	const ngramEncoding = 8
-	for i := 0; i < len(textContent); i += ngramEncoding {
-		j := i / ngramEncoding
-		ng := ngram(binary.BigEndian.Uint64(textContent[i : i+ngramEncoding]))
-		d.ngrams[ng] = simpleSection{
-			toc.postings.data.off + postingsIndex[j],
-			postingsIndex[j+1] - postingsIndex[j],
-		}
-	}
 
 	d.fileBranchMasks, err = readSectionU64(d.file, toc.branchMasks)
 	if err != nil {
@@ -211,25 +200,11 @@
 
 	d.fileNameIndex = toc.fileNames.relativeIndex()
 
-	nameNgramText, err := d.readSectionBlob(toc.nameNgramText)
+	d.fileNameNgrams, err = d.readFileNameNgrams(toc)
 	if err != nil {
 		return nil, err
 	}
 
-	fileNamePostingsData, err := d.readSectionBlob(toc.namePostings.data)
-	if err != nil {
-		return nil, err
-	}
-
-	fileNamePostingsIndex := toc.namePostings.relativeIndex()
-	for i := 0; i < len(nameNgramText); i += ngramEncoding {
-		j := i / ngramEncoding
-		off := fileNamePostingsIndex[j]
-		end := fileNamePostingsIndex[j+1]
-		ng := ngram(binary.BigEndian.Uint64(nameNgramText[i : i+ngramEncoding]))
-		d.fileNameNgrams[ng] = fromDeltas(fileNamePostingsData[off:end], nil)
-	}
-
 	for j, br := range d.repoMetaData.Branches {
 		id := uint(1) << uint(j)
 		d.branchIDs[br.Name] = id
@@ -276,6 +251,53 @@
 	return &d, nil
 }
 
+const ngramEncoding = 8
+
+func (d *indexData) readNgrams(toc *indexTOC) (map[ngram]simpleSection, error) {
+	textContent, err := d.readSectionBlob(toc.ngramText)
+	if err != nil {
+		return nil, err
+	}
+	postingsIndex := toc.postings.relativeIndex()
+
+	ngrams := map[ngram]simpleSection{}
+	for i := 0; i < len(textContent); i += ngramEncoding {
+		j := i / ngramEncoding
+		ng := ngram(binary.BigEndian.Uint64(textContent[i : i+ngramEncoding]))
+		ngrams[ng] = simpleSection{
+			toc.postings.data.off + postingsIndex[j],
+			postingsIndex[j+1] - postingsIndex[j],
+		}
+	}
+
+	return ngrams, nil
+}
+
+func (d *indexData) readFileNameNgrams(toc *indexTOC) (map[ngram][]uint32, error) {
+	nameNgramText, err := d.readSectionBlob(toc.nameNgramText)
+	if err != nil {
+		return nil, err
+	}
+
+	fileNamePostingsData, err := d.readSectionBlob(toc.namePostings.data)
+	if err != nil {
+		return nil, err
+	}
+
+	fileNamePostingsIndex := toc.namePostings.relativeIndex()
+
+	fileNameNgrams := map[ngram][]uint32{}
+	for i := 0; i < len(nameNgramText); i += ngramEncoding {
+		j := i / ngramEncoding
+		off := fileNamePostingsIndex[j]
+		end := fileNamePostingsIndex[j+1]
+		ng := ngram(binary.BigEndian.Uint64(nameNgramText[i : i+ngramEncoding]))
+		fileNameNgrams[ng] = fromDeltas(fileNamePostingsData[off:end], nil)
+	}
+
+	return fileNameNgrams, nil
+}
+
 func (d *indexData) verify() error {
 	// This is not an exhaustive check: the postings can easily
 	// generate OOB acccesses, and are expensive to check, but this lets us rule out