web: serve robots.txt, disallowing /search
The example queries on the entry page are not limited by repository,
so they may touch a large amount of memory. For public servers,
avoiding robots may increase availability. Anecdotally, cs.bazel.build
is having fits of crashing, ostensibly caused by crawlers overloading
the search engine
Change-Id: I4417c9389ce57f4d5ebee7695397e20540d15779
diff --git a/web/e2e_test.go b/web/e2e_test.go
index dc8531f..479441b 100644
--- a/web/e2e_test.go
+++ b/web/e2e_test.go
@@ -116,6 +116,9 @@
"/search?q=magic": {
`value=magic`,
},
+ "/robots.txt": {
+ "disallow: /search",
+ },
} {
checkNeedles(t, ts, req, needles)
}
diff --git a/web/server.go b/web/server.go
index ef08e98..94a16ac 100644
--- a/web/server.go
+++ b/web/server.go
@@ -106,6 +106,7 @@
result *template.Template
print *template.Template
about *template.Template
+ robots *template.Template
startTime time.Time
@@ -147,6 +148,7 @@
"search": &s.search,
"repolist": &s.repolist,
"about": &s.about,
+ "robots": &s.robots,
} {
*v = s.Top.Lookup(k)
if *v == nil {
@@ -160,6 +162,7 @@
mux := http.NewServeMux()
if s.HTML {
+ mux.HandleFunc("/robots.txt", s.serveRobots)
mux.HandleFunc("/search", s.serveSearch)
mux.HandleFunc("/", s.serveSearchBox)
mux.HandleFunc("/about", s.serveAbout)
@@ -394,6 +397,22 @@
}
}
+func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
+ data := struct{}{}
+ var buf bytes.Buffer
+ if err := s.robots.Execute(&buf, &data); err != nil {
+ return err
+ }
+ w.Write(buf.Bytes())
+ return nil
+}
+
+func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
+ if err := s.serveRobotsErr(w, r); err != nil {
+ http.Error(w, err.Error(), http.StatusTeapot)
+ }
+}
+
func (s *Server) serveListReposErr(q query.Q, qStr string, w http.ResponseWriter, r *http.Request) error {
ctx := r.Context()
repos, err := s.Searcher.List(ctx, q)
diff --git a/web/templates.go b/web/templates.go
index 5f1adec..eea9bb7 100644
--- a/web/templates.go
+++ b/web/templates.go
@@ -393,6 +393,10 @@
</div>
</nav>
`,
+ "robots": `
+user-agent: *
+disallow: /search
+`,
}
func init() {
