Add support for robots.txt user agent control from web crawlers (#10)

1 files changed, 7 insertions, 3 deletions

diff --git a/cmd/gopherproxy/main.go b/cmd/gopherproxy/main.go
index 9a7d1f9..84bde30 100644
--- a/cmd/gopherproxy/main.go
+++ b/cmd/gopherproxy/main.go
@@ -8,12 +8,16 @@ import (
 )
 
 var (
-        bind = flag.String("bind", "0.0.0.0:8000", "[int]:port to bind to")
-        uri  = flag.String("uri", "floodgap.com", "<host>:[port] to proxy to")
+        // TODO: Allow config file and environment vars
+        //       (opt -> env -> config -> default)
+        bind       = flag.String("bind", "0.0.0.0:8000", "[int]:port to bind to")
+        robotsfile = flag.String("robots-file", "robots.txt", "robots.txt file")
+        uri        = flag.String("uri", "floodgap.com", "<host>:[port] to proxy to")
 )
 
 func main() {
         flag.Parse()
 
-        log.Fatal(gopherproxy.ListenAndServe(*bind, *uri))
+        // Use a config struct
+        log.Fatal(gopherproxy.ListenAndServe(*bind, *robotsfile, *uri))
 }