From de13213492ce02464c5ecd8544ea5ae6b33fa712 Mon Sep 17 00:00:00 2001 From: Feuerfuchs Date: Thu, 20 Jun 2019 15:47:04 +0200 Subject: Make output about ignored robots.txt optional --- cmd/gopherproxy/main.go | 9 +++++---- gopherproxy.go | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/gopherproxy/main.go b/cmd/gopherproxy/main.go index aced41b..e9dea18 100644 --- a/cmd/gopherproxy/main.go +++ b/cmd/gopherproxy/main.go @@ -10,14 +10,15 @@ import ( var ( // TODO: Allow config file and environment vars // (opt -> env -> config -> default) - bind = flag.String("bind", "0.0.0.0:8000", "[int]:port to bind to") - robotsfile = flag.String("robots-file", "robots.txt", "robots.txt file") - uri = flag.String("uri", "floodgap.com", ":[port] to proxy to") + bind = flag.String("bind", "0.0.0.0:8000", "[int]:port to bind to") + robotsfile = flag.String("robots-file", "robots.txt", "robots.txt file") + robotsdebug = flag.Bool("robots-debug", false, "print output about ignored robots.txt") + uri = flag.String("uri", "floodgap.com", ":[port] to proxy to") ) func main() { flag.Parse() // Use a config struct - log.Fatal(gopherproxy.ListenAndServe(*bind, *robotsfile, *uri)) + log.Fatal(gopherproxy.ListenAndServe(*bind, *robotsfile, *robotsdebug, *uri)) } diff --git a/gopherproxy.go b/gopherproxy.go index 74f33bb..961806a 100644 --- a/gopherproxy.go +++ b/gopherproxy.go @@ -87,12 +87,12 @@ func renderDirectory(w http.ResponseWriter, tpl *template.Template, styletext st // to the request path and renders the content using the provided template. // The optional robots parameters points to a robotstxt.RobotsData struct // to test user agents against a configurable robotst.txt file. -func GopherHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, styletext string, uri string) http.HandlerFunc { +func GopherHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, robotsdebug bool, styletext string, uri string) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { agent := req.UserAgent() path := strings.TrimPrefix(req.URL.Path, "/") - if robotsdata != nil && !robotsdata.TestAgent(path, agent) { + if robotsdata != nil && robotsdebug && !robotsdata.TestAgent(path, agent) { log.Printf("UserAgent %s ignored robots.txt", agent) } @@ -203,7 +203,7 @@ func FontHandler(woff2 bool, fontdata []byte) http.HandlerFunc { // specified by the request. The robots argument is a pointer to // a robotstxt.RobotsData struct for testing user agents against // a configurable robots.txt file. -func ListenAndServe(bind, robotsfile, uri string) error { +func ListenAndServe(bind, robotsfile string, robotsdebug bool, uri string) error { var ( tpl *template.Template robotsdata *robotstxt.RobotsData @@ -265,7 +265,7 @@ func ListenAndServe(bind, robotsfile, uri string) error { log.Fatal(err) } - http.HandleFunc("/", GopherHandler(tpl, robotsdata, styletext, uri)) + http.HandleFunc("/", GopherHandler(tpl, robotsdata, robotsdebug, styletext, uri)) http.HandleFunc("/robots.txt", RobotsTxtHandler(robotstxtdata)) http.HandleFunc("/favicon.ico", FaviconHandler(favicondata)) http.HandleFunc("/iosevka-term-ss03-regular.woff", FontHandler(false, fontdataw)) -- cgit v1.2.3-70-g09d2