From 93115f804220c31c2aa10f123560fb11135f06d8 Mon Sep 17 00:00:00 2001 From: Feuerfuchs Date: Tue, 26 Nov 2019 13:13:02 +0100 Subject: Add IPv6 support, general restructuring --- .gitignore | 2 +- Makefile | 4 +- cmd/gopherproxy/main.go | 25 -- go.mod | 1 - gopherproxy.go | 693 ------------------------------------ gopherproxy/gopherproxy.go | 697 +++++++++++++++++++++++++++++++++++++ gopherproxy/libgemini/libgemini.go | 145 ++++++++ gopherproxy/libgopher/libgopher.go | 312 +++++++++++++++++ gopherproxy/template.go | 122 +++++++ libgemini.go | 153 -------- main.go | 25 ++ template.go | 122 ------- 12 files changed, 1304 insertions(+), 997 deletions(-) delete mode 100644 cmd/gopherproxy/main.go delete mode 100644 gopherproxy.go create mode 100644 gopherproxy/gopherproxy.go create mode 100644 gopherproxy/libgemini/libgemini.go create mode 100644 gopherproxy/libgopher/libgopher.go create mode 100644 gopherproxy/template.go delete mode 100644 libgemini.go create mode 100644 main.go delete mode 100644 template.go diff --git a/.gitignore b/.gitignore index e036b02..fe92af3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ dist *.bak coverage.txt -gopherproxy +gopherproxy.bin .vscode diff --git a/Makefile b/Makefile index 1d65cf0..75a8f2e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: dev dev: build - ./gopherproxy -bind 127.0.0.1:8000 + ./gopherproxy.bin -bind 127.0.0.1:8000 build: clean sassc -t compressed css/main.scss assets/style.css @@ -12,7 +12,7 @@ build: clean pyftsubset fonts/iosevka-term-ss03-regular.ttf --name-IDs+=0,4,6 --text-file=fonts/glyphs.txt --flavor='woff2' --output-file='assets/iosevka-term-ss03-regular.woff2' pyftsubset fonts/iosevka-aile-regular.ttf --name-IDs+=0,4,6 --text-file=fonts/glyphs.txt --flavor='woff' --with-zopfli --output-file='assets/iosevka-aile-regular.woff' pyftsubset fonts/iosevka-aile-regular.ttf --name-IDs+=0,4,6 --text-file=fonts/glyphs.txt --flavor='woff2' --output-file='assets/iosevka-aile-regular.woff2' - go build -o ./gopherproxy ./cmd/gopherproxy/main.go + go build -o ./gopherproxy.bin ./main.go profile: @go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench . diff --git a/cmd/gopherproxy/main.go b/cmd/gopherproxy/main.go deleted file mode 100644 index 6e6f48c..0000000 --- a/cmd/gopherproxy/main.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "flag" - "log" - - "git.feuerfuchs.dev/Feuerfuchs/gopherproxy" -) - -var ( - // TODO: Allow config file and environment vars - // (opt -> env -> config -> default) - bind = flag.String("bind", "0.0.0.0:8000", "[int]:port to bind to") - startpagefile = flag.String("startpage-file", "startpage.txt", "Default page to display if no URL is specified") - robotsfile = flag.String("robots-file", "robots.txt", "robots.txt file") - robotsdebug = flag.Bool("robots-debug", false, "print output about ignored robots.txt") - vipsconcurrency = flag.Int("vips-concurrency", 1, "Concurrency level of libvips") -) - -func main() { - flag.Parse() - - // Use a config struct - log.Fatal(gopherproxy.ListenAndServe(*bind, *startpagefile, *robotsfile, *robotsdebug, *vipsconcurrency)) -} diff --git a/go.mod b/go.mod index 503bb87..6cde9a1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ require ( github.com/davidbyttow/govips v0.0.0-20190304175058-d272f04c0fea github.com/gobuffalo/packr/v2 v2.1.0 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 - github.com/prologic/go-gopher v0.0.0-20181230133552-0c68ed5f58b0 github.com/temoto/robotstxt v0.0.0-20180810133444-97ee4a9ee6ea golang.org/x/net v0.0.0-20190311183353-d8887717615a golang.org/x/text v0.3.0 diff --git a/gopherproxy.go b/gopherproxy.go deleted file mode 100644 index 8c0bb89..0000000 --- a/gopherproxy.go +++ /dev/null @@ -1,693 +0,0 @@ -package gopherproxy - -import ( - "bufio" - "bytes" - "crypto/md5" - "fmt" - "html" - "html/template" - "io" - "io/ioutil" - "log" - "mime" - "net/http" - "net/url" - "regexp" - "strings" - - "golang.org/x/net/html/charset" - "golang.org/x/text/transform" - - "github.com/temoto/robotstxt" - - "github.com/prologic/go-gopher" - - "github.com/gobuffalo/packr/v2" - - "github.com/davidbyttow/govips/pkg/vips" - - "github.com/NYTimes/gziphandler" -) - -const ( - ITEM_TYPE_GEMINI_LINE = "" - ITEM_TYPE_GEMINI_LINK = " =>" -) - -type Item struct { - Link template.URL - Type string - Text string -} - -type AssetList struct { - Style string - JS string - FontW string - FontW2 string - PropFontW string - PropFontW2 string -} - -func resolveURI(uri string, baseURL *url.URL) (resolvedURI string) { - if strings.HasPrefix(uri, "//") { - resolvedURI = "/gemini/" + strings.TrimPrefix(uri, "//") - } else if strings.HasPrefix(uri, "gemini://") { - resolvedURI = "/gemini/" + strings.TrimPrefix(uri, "gemini://") - } else if strings.HasPrefix(uri, "gopher://") { - resolvedURI = "/gopher/" + strings.TrimPrefix(uri, "gopher://") - } else { - url, err := url.Parse(uri) - if err != nil { - return "" - } - adjustedURI := baseURL.ResolveReference(url) - if adjustedURI.Scheme == "gemini" { - resolvedURI = "/gemini/" + adjustedURI.Host + adjustedURI.Path - } else if adjustedURI.Scheme == "gopher" { - resolvedURI = "/gopher/" + adjustedURI.Host + adjustedURI.Path - } else { - resolvedURI = adjustedURI.String() - } - } - - return -} - -func renderGopherDirectory(w http.ResponseWriter, tpl *template.Template, assetList AssetList, uri string, hostport string, d gopher.Directory) error { - var title string - - out := make([]Item, len(d.Items)) - - for i, x := range d.Items { - if x.Type == gopher.INFO && x.Selector == "TITLE" { - title = x.Description - continue - } - - tr := Item{ - Text: x.Description, - Type: x.Type.String(), - } - - if x.Type == gopher.INFO { - out[i] = tr - continue - } - - if strings.HasPrefix(x.Selector, "URL:") || strings.HasPrefix(x.Selector, "/URL:") { - link := strings.TrimPrefix(strings.TrimPrefix(x.Selector, "/"), "URL:") - if strings.HasPrefix(link, "gemini://") { - link = fmt.Sprintf( - "/gemini/%s", - strings.TrimPrefix(link, "gemini://"), - ) - } else if strings.HasPrefix(link, "gopher://") { - link = fmt.Sprintf( - "/gopher/%s", - strings.TrimPrefix(link, "gopher://"), - ) - } - tr.Link = template.URL(link) - } else { - var hostport string - if x.Port == 70 { - hostport = x.Host - } else { - hostport = fmt.Sprintf("%s:%d", x.Host, x.Port) - } - path := url.PathEscape(x.Selector) - path = strings.Replace(path, "%2F", "/", -1) - tr.Link = template.URL( - fmt.Sprintf( - "/gopher/%s/%s%s", - hostport, - string(byte(x.Type)), - path, - ), - ) - } - - out[i] = tr - } - - if title == "" { - if uri != "" { - title = fmt.Sprintf("%s/%s", hostport, uri) - } else { - title = hostport - } - } - - return tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - Lines []Item - RawText string - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, out, "", false, "gopher"}) -} - -func parseGeminiDocument(body *bytes.Buffer, uri string, hostport string) (items []Item) { - baseURL, err := url.Parse(fmt.Sprintf( - "gemini://%s/%s", - hostport, - uri, - )) - if err != nil { - return []Item{} - } - - scanner := bufio.NewScanner(body) - - for scanner.Scan() { - line := strings.Trim(scanner.Text(), "\r\n") - - item := Item{ - Type: ITEM_TYPE_GEMINI_LINE, - Text: line, - } - - linkMatch := GeminiLinkPattern.FindStringSubmatch(line) - if len(linkMatch) != 0 && linkMatch[0] != "" { - item.Type = ITEM_TYPE_GEMINI_LINK - item.Link = template.URL(resolveURI(linkMatch[1], baseURL)) - item.Text = linkMatch[2] - if item.Text == "" { - item.Text = linkMatch[1] - } - } - - items = append(items, item) - } - - return -} - -func DefaultHandler(tpl *template.Template, startpagetext string, assetList AssetList) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - if err := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{"Gopher/Gemini proxy", "", assetList, startpagetext, nil, false, "startpage"}); err != nil { - log.Println("Template error: " + err.Error()) - } - } -} - -// GopherHandler returns a Handler that proxies requests -// to the specified Gopher server as denoated by the first argument -// to the request path and renders the content using the provided template. -// The optional robots parameters points to a robotstxt.RobotsData struct -// to test user agents against a configurable robotst.txt file. -func GopherHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, assetList AssetList, robotsdebug bool) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - agent := req.UserAgent() - path := strings.TrimPrefix(req.URL.Path, "/gopher/") - - if robotsdata != nil && robotsdebug && !robotsdata.TestAgent(path, agent) { - log.Printf("UserAgent %s ignored robots.txt", agent) - } - - parts := strings.Split(path, "/") - hostport := parts[0] - - if len(hostport) == 0 { - http.Redirect(w, req, "/", http.StatusFound) - return - } - - title := hostport - - var qs string - - if req.URL.RawQuery != "" { - qs = fmt.Sprintf("?%s", url.QueryEscape(req.URL.RawQuery)) - } - - uri, err := url.QueryUnescape(strings.Join(parts[1:], "/")) - if err != nil { - if e := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, hostport, assetList, fmt.Sprintf("Error: %s", err), nil, true, "gopher"}); e != nil { - log.Println("Template error: " + e.Error()) - log.Println(err.Error()) - } - return - } - - if uri != "" { - title = fmt.Sprintf("%s/%s", hostport, uri) - } - - res, err := gopher.Get( - fmt.Sprintf( - "gopher://%s/%s%s", - hostport, - uri, - qs, - ), - ) - - if err != nil { - if e := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gopher"}); e != nil { - log.Println("Template error: " + e.Error()) - } - return - } - - if res.Body != nil { - if len(parts) < 2 { - io.Copy(w, res.Body) - } else if strings.HasPrefix(parts[1], "0") && !strings.HasSuffix(uri, ".xml") && !strings.HasSuffix(uri, ".asc") { - buf := new(bytes.Buffer) - buf.ReadFrom(res.Body) - if err := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, buf.String(), nil, false, "gopher"}); err != nil { - log.Println("Template error: " + err.Error()) - } - } else if strings.HasPrefix(parts[1], "T") { - _, _, err = vips.NewTransform(). - Load(res.Body). - ResizeStrategy(vips.ResizeStrategyAuto). - ResizeWidth(160). - Quality(75). - Output(w). - Apply() - } else { - io.Copy(w, res.Body) - } - } else { - if err := renderGopherDirectory(w, tpl, assetList, uri, hostport, res.Dir); err != nil { - if e := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gopher"}); e != nil { - log.Println("Template error: " + e.Error()) - log.Println(e.Error()) - } - } - } - } -} - -func GeminiHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, assetList AssetList, robotsdebug bool) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - agent := req.UserAgent() - path := strings.TrimPrefix(req.URL.Path, "/gemini/") - - if robotsdata != nil && robotsdebug && !robotsdata.TestAgent(path, agent) { - log.Printf("UserAgent %s ignored robots.txt", agent) - } - - parts := strings.Split(path, "/") - hostport := parts[0] - - if len(hostport) == 0 { - http.Redirect(w, req, "/", http.StatusFound) - return - } - - title := hostport - - var qs string - - if req.URL.RawQuery != "" { - qs = fmt.Sprintf("?%s", url.QueryEscape(req.URL.RawQuery)) - } - - uri, err := url.QueryUnescape(strings.Join(parts[1:], "/")) - if err != nil { - if e := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, hostport, assetList, fmt.Sprintf("Error: %s", err), nil, true, "gemini"}); e != nil { - log.Println("Template error: " + e.Error()) - log.Println(err.Error()) - } - return - } - - if uri != "" { - title = fmt.Sprintf("%s/%s", hostport, uri) - } - - res, err := GeminiGet( - fmt.Sprintf( - "gemini://%s/%s%s", - hostport, - uri, - qs, - ), - ) - - if err != nil { - if e := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gemini"}); e != nil { - log.Println("Template error: " + e.Error()) - log.Println(err.Error()) - } - return - } - - if int(res.Header.Status/10) == 3 { - baseURL, err := url.Parse(fmt.Sprintf( - "gemini://%s/%s", - hostport, - uri, - )) - if err != nil { - if e := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gemini"}); e != nil { - log.Println("Template error: " + e.Error()) - log.Println(err.Error()) - } - return - } - - http.Redirect(w, req, resolveURI(res.Header.Meta, baseURL), http.StatusFound) - return - } - - if int(res.Header.Status/10) != 2 { - if err := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error %d: %s", res.Header.Status, res.Header.Meta), nil, true, "gemini"}); err != nil { - log.Println("Template error: " + err.Error()) - } - return - } - - if strings.HasPrefix(res.Header.Meta, "text/") { - buf := new(bytes.Buffer) - - _, params, err := mime.ParseMediaType(res.Header.Meta) - if err != nil { - buf.ReadFrom(res.Body) - } else { - encoding, _ := charset.Lookup(params["charset"]) - readbuf := new(bytes.Buffer) - readbuf.ReadFrom(res.Body) - - writer := transform.NewWriter(buf, encoding.NewDecoder()) - writer.Write(readbuf.Bytes()) - writer.Close() - } - - var ( - rawText string - items []Item - ) - - if strings.HasPrefix(res.Header.Meta, MIME_GEMINI) { - items = parseGeminiDocument(buf, uri, hostport) - } else { - rawText = buf.String() - } - - if err := tpl.Execute(w, struct { - Title string - URI string - Assets AssetList - RawText string - Lines []Item - Error bool - Protocol string - }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, rawText, items, false, "gemini"}); err != nil { - log.Println("Template error: " + err.Error()) - } - } else { - io.Copy(w, res.Body) - } - } -} - -// RobotsTxtHandler returns the contents of the robots.txt file -// if configured and valid. -func RobotsTxtHandler(robotstxtdata []byte) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - if robotstxtdata == nil { - http.Error(w, "Not Found", http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "text/plain") - w.Write(robotstxtdata) - } -} - -func FaviconHandler(favicondata []byte) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - if favicondata == nil { - http.Error(w, "Not Found", http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "image/vnd.microsoft.icon") - w.Header().Set("Cache-Control", "max-age=2592000") - w.Write(favicondata) - } -} - -func StyleHandler(styledata []byte) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "text/css") - w.Header().Set("Cache-Control", "max-age=2592000") - w.Write(styledata) - } -} - -func JavaScriptHandler(jsdata []byte) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "text/javascript") - w.Header().Set("Cache-Control", "max-age=2592000") - w.Write(jsdata) - } -} - -func FontHandler(woff2 bool, fontdata []byte) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - if fontdata == nil { - http.Error(w, "Not Found", http.StatusNotFound) - return - } - - if woff2 { - w.Header().Set("Content-Type", "font/woff2") - } else { - w.Header().Set("Content-Type", "font/woff") - } - w.Header().Set("Cache-Control", "max-age=2592000") - - w.Write(fontdata) - } -} - -// ListenAndServe creates a listening HTTP server bound to -// the interface specified by bind and sets up a Gopher to HTTP -// proxy proxying requests as requested and by default will prozy -// to a Gopher server address specified by uri if no servers is -// specified by the request. The robots argument is a pointer to -// a robotstxt.RobotsData struct for testing user agents against -// a configurable robots.txt file. -func ListenAndServe(bind, startpagefile string, robotsfile string, robotsdebug bool, vipsconcurrency int) error { - var ( - tpl *template.Template - robotsdata *robotstxt.RobotsData - ) - - robotstxtdata, err := ioutil.ReadFile(robotsfile) - if err != nil { - log.Printf("error reading robots.txt: %s", err) - robotstxtdata = nil - } else { - robotsdata, err = robotstxt.FromBytes(robotstxtdata) - if err != nil { - log.Printf("error reading robots.txt: %s", err) - robotstxtdata = nil - } - } - - box := packr.New("assets", "./assets") - - fontdataw, err := box.Find("iosevka-term-ss03-regular.woff") - if err != nil { - fontdataw = []byte{} - } - fontwAsset := fmt.Sprintf("/iosevka-term-ss03-regular-%x.woff", md5.Sum(fontdataw)) - - fontdataw2, err := box.Find("iosevka-term-ss03-regular.woff2") - if err != nil { - fontdataw2 = []byte{} - } - fontw2Asset := fmt.Sprintf("/iosevka-term-ss03-regular-%x.woff2", md5.Sum(fontdataw2)) - - propfontdataw, err := box.Find("iosevka-aile-regular.woff") - if err != nil { - propfontdataw = []byte{} - } - propfontwAsset := fmt.Sprintf("/iosevka-aile-regular-%x.woff", md5.Sum(propfontdataw)) - - propfontdataw2, err := box.Find("iosevka-aile-regular.woff2") - if err != nil { - propfontdataw2 = []byte{} - } - propfontw2Asset := fmt.Sprintf("/iosevka-aile-regular-%x.woff2", md5.Sum(propfontdataw2)) - - styledata, err := box.Find("style.css") - if err != nil { - styledata = []byte{} - } - styleAsset := fmt.Sprintf("/style-%x.css", md5.Sum(styledata)) - - jsdata, err := box.Find("main.js") - if err != nil { - jsdata = []byte{} - } - jsAsset := fmt.Sprintf("/main-%x.js", md5.Sum(jsdata)) - - favicondata, err := box.Find("favicon.ico") - if err != nil { - favicondata = []byte{} - } - - startpagedata, err := ioutil.ReadFile(startpagefile) - if err != nil { - startpagedata, err = box.Find("startpage.txt") - if err != nil { - startpagedata = []byte{} - } - } - startpagetext := string(startpagedata) - - tpldata, err := ioutil.ReadFile(".template") - if err == nil { - tpltext = string(tpldata) - } - - funcMap := template.FuncMap{ - "safeHtml": func(s string) template.HTML { - return template.HTML(s) - }, - "safeCss": func(s string) template.CSS { - return template.CSS(s) - }, - "safeJs": func(s string) template.JS { - return template.JS(s) - }, - "HTMLEscape": func(s string) string { - return html.EscapeString(s) - }, - "split": strings.Split, - "last": func(s []string) string { - return s[len(s)-1] - }, - "pop": func(s []string) []string { - return s[:len(s)-1] - }, - "replace": func(pattern, output string, input interface{}) string { - var re = regexp.MustCompile(pattern) - var inputStr = fmt.Sprintf("%v", input) - return re.ReplaceAllString(inputStr, output) - }, - "trimLeftChar": func(s string) string { - for i := range s { - if i > 0 { - return s[i:] - } - } - return s[:0] - }, - "hasPrefix": func(s string, prefix string) bool { - return strings.HasPrefix(s, prefix) - }, - "title": func(s string) string { - return strings.Title(s) - }, - } - - tpl, err = template.New("gophermenu").Funcs(funcMap).Parse(tpltext) - if err != nil { - log.Fatal(err) - } - - vips.Startup(&vips.Config{ - ConcurrencyLevel: vipsconcurrency, - }) - - assets := AssetList{styleAsset, jsAsset, fontwAsset, fontw2Asset, propfontwAsset, propfontw2Asset} - - http.Handle("/", gziphandler.GzipHandler(DefaultHandler(tpl, startpagetext, assets))) - http.Handle("/gopher/", gziphandler.GzipHandler(GopherHandler(tpl, robotsdata, assets, robotsdebug))) - http.Handle("/gemini/", gziphandler.GzipHandler(GeminiHandler(tpl, robotsdata, assets, robotsdebug))) - http.Handle("/robots.txt", gziphandler.GzipHandler(RobotsTxtHandler(robotstxtdata))) - http.Handle("/favicon.ico", gziphandler.GzipHandler(FaviconHandler(favicondata))) - http.Handle(styleAsset, gziphandler.GzipHandler(StyleHandler(styledata))) - http.Handle(jsAsset, gziphandler.GzipHandler(JavaScriptHandler(jsdata))) - http.HandleFunc(fontwAsset, FontHandler(false, fontdataw)) - http.HandleFunc(fontw2Asset, FontHandler(true, fontdataw2)) - http.HandleFunc(propfontwAsset, FontHandler(false, propfontdataw)) - http.HandleFunc(propfontw2Asset, FontHandler(true, propfontdataw2)) - //http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("assets/")))) - - return http.ListenAndServe(bind, nil) -} diff --git a/gopherproxy/gopherproxy.go b/gopherproxy/gopherproxy.go new file mode 100644 index 0000000..6556845 --- /dev/null +++ b/gopherproxy/gopherproxy.go @@ -0,0 +1,697 @@ +package gopherproxy + +import ( + "bufio" + "bytes" + "crypto/md5" + "fmt" + "html" + "html/template" + "io" + "io/ioutil" + "log" + "mime" + "net" + "net/http" + "net/url" + "regexp" + "strings" + + "golang.org/x/net/html/charset" + "golang.org/x/text/transform" + + "git.feuerfuchs.dev/Feuerfuchs/gopherproxy/gopherproxy/libgemini" + "git.feuerfuchs.dev/Feuerfuchs/gopherproxy/gopherproxy/libgopher" + + "github.com/NYTimes/gziphandler" + "github.com/davidbyttow/govips/pkg/vips" + "github.com/gobuffalo/packr/v2" + "github.com/temoto/robotstxt" +) + +const ( + ITEM_TYPE_GEMINI_LINE = "" + ITEM_TYPE_GEMINI_LINK = " =>" +) + +type Item struct { + Link template.URL + Type string + Text string +} + +type AssetList struct { + Style string + JS string + FontW string + FontW2 string + PropFontW string + PropFontW2 string +} + +func resolveURI(uri string, baseURL *url.URL) (resolvedURI string) { + if strings.HasPrefix(uri, "//") { + resolvedURI = "/gemini/" + strings.TrimPrefix(uri, "//") + } else if strings.HasPrefix(uri, "gemini://") { + resolvedURI = "/gemini/" + strings.TrimPrefix(uri, "gemini://") + } else if strings.HasPrefix(uri, "gopher://") { + resolvedURI = "/gopher/" + strings.TrimPrefix(uri, "gopher://") + } else { + url, err := url.Parse(uri) + if err != nil { + return "" + } + adjustedURI := baseURL.ResolveReference(url) + path := adjustedURI.Path + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if adjustedURI.Scheme == "gemini" { + resolvedURI = "/gemini/" + adjustedURI.Host + path + } else if adjustedURI.Scheme == "gopher" { + resolvedURI = "/gopher/" + adjustedURI.Host + path + } else { + resolvedURI = adjustedURI.String() + } + } + + return +} + +func renderGopherDirectory(w http.ResponseWriter, tpl *template.Template, assetList AssetList, uri string, hostport string, d libgopher.Directory) error { + var title string + + out := make([]Item, len(d.Items)) + + for i, x := range d.Items { + if x.Type == libgopher.INFO && x.Selector == "TITLE" { + title = x.Description + continue + } + + tr := Item{ + Text: x.Description, + Type: x.Type.String(), + } + + if x.Type == libgopher.INFO { + out[i] = tr + continue + } + + if strings.HasPrefix(x.Selector, "URL:") || strings.HasPrefix(x.Selector, "/URL:") { + link := strings.TrimPrefix(strings.TrimPrefix(x.Selector, "/"), "URL:") + if strings.HasPrefix(link, "gemini://") { + link = fmt.Sprintf( + "/gemini/%s", + strings.TrimPrefix(link, "gemini://"), + ) + } else if strings.HasPrefix(link, "gopher://") { + link = fmt.Sprintf( + "/gopher/%s", + strings.TrimPrefix(link, "gopher://"), + ) + } + tr.Link = template.URL(link) + } else { + var linkHostport string + if x.Port != "70" { + linkHostport = net.JoinHostPort(x.Host, x.Port) + } else { + linkHostport = x.Host + } + + path := url.PathEscape(x.Selector) + path = strings.Replace(path, "%2F", "/", -1) + tr.Link = template.URL( + fmt.Sprintf( + "/gopher/%s/%s%s", + linkHostport, + string(byte(x.Type)), + path, + ), + ) + } + + out[i] = tr + } + + if title == "" { + if uri != "" { + title = fmt.Sprintf("%s/%s", hostport, uri) + } else { + title = hostport + } + } + + return tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + Lines []Item + RawText string + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, out, "", false, "gopher"}) +} + +func parseGeminiDocument(body *bytes.Buffer, uri string, hostport string) (items []Item) { + baseURL, err := url.Parse(fmt.Sprintf( + "gemini://%s/%s", + hostport, + uri, + )) + if err != nil { + return []Item{} + } + + scanner := bufio.NewScanner(body) + + for scanner.Scan() { + line := strings.Trim(scanner.Text(), "\r\n") + + item := Item{ + Type: ITEM_TYPE_GEMINI_LINE, + Text: line, + } + + linkMatch := libgemini.LinkPattern.FindStringSubmatch(line) + if len(linkMatch) != 0 && linkMatch[0] != "" { + item.Type = ITEM_TYPE_GEMINI_LINK + item.Link = template.URL(resolveURI(linkMatch[1], baseURL)) + item.Text = linkMatch[2] + if item.Text == "" { + item.Text = linkMatch[1] + } + } + + items = append(items, item) + } + + return +} + +func DefaultHandler(tpl *template.Template, startpagetext string, assetList AssetList) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if err := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{"Gopher/Gemini proxy", "", assetList, startpagetext, nil, false, "startpage"}); err != nil { + log.Println("Template error: " + err.Error()) + } + } +} + +// GopherHandler returns a Handler that proxies requests +// to the specified Gopher server as denoated by the first argument +// to the request path and renders the content using the provided template. +// The optional robots parameters points to a robotstxt.RobotsData struct +// to test user agents against a configurable robotst.txt file. +func GopherHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, assetList AssetList, robotsdebug bool) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + agent := req.UserAgent() + path := strings.TrimPrefix(req.URL.Path, "/gopher/") + + if robotsdata != nil && robotsdebug && !robotsdata.TestAgent(path, agent) { + log.Printf("UserAgent %s ignored robots.txt", agent) + } + + parts := strings.Split(path, "/") + hostport := parts[0] + + if len(hostport) == 0 { + http.Redirect(w, req, "/", http.StatusFound) + return + } + + title := hostport + + var qs string + + if req.URL.RawQuery != "" { + qs = fmt.Sprintf("?%s", url.QueryEscape(req.URL.RawQuery)) + } + + uri, err := url.QueryUnescape(strings.Join(parts[1:], "/")) + if err != nil { + if e := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, hostport, assetList, fmt.Sprintf("Error: %s", err), nil, true, "gopher"}); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + if uri != "" { + title = fmt.Sprintf("%s/%s", hostport, uri) + } + + res, err := libgopher.Get( + fmt.Sprintf( + "gopher://%s/%s%s", + hostport, + uri, + qs, + ), + ) + + if err != nil { + if e := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gopher"}); e != nil { + log.Println("Template error: " + e.Error()) + } + return + } + + if res.Body != nil { + if len(parts) < 2 { + io.Copy(w, res.Body) + } else if strings.HasPrefix(parts[1], "0") && !strings.HasSuffix(uri, ".xml") && !strings.HasSuffix(uri, ".asc") { + buf := new(bytes.Buffer) + buf.ReadFrom(res.Body) + if err := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, buf.String(), nil, false, "gopher"}); err != nil { + log.Println("Template error: " + err.Error()) + } + } else if strings.HasPrefix(parts[1], "T") { + _, _, err = vips.NewTransform(). + Load(res.Body). + ResizeStrategy(vips.ResizeStrategyAuto). + ResizeWidth(160). + Quality(75). + Output(w). + Apply() + } else { + io.Copy(w, res.Body) + } + } else { + if err := renderGopherDirectory(w, tpl, assetList, uri, hostport, res.Dir); err != nil { + if e := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gopher"}); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(e.Error()) + } + } + } + } +} + +func GeminiHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, assetList AssetList, robotsdebug bool) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + agent := req.UserAgent() + path := strings.TrimPrefix(req.URL.Path, "/gemini/") + + if robotsdata != nil && robotsdebug && !robotsdata.TestAgent(path, agent) { + log.Printf("UserAgent %s ignored robots.txt", agent) + } + + parts := strings.Split(path, "/") + hostport := parts[0] + + if len(hostport) == 0 { + http.Redirect(w, req, "/", http.StatusFound) + return + } + + title := hostport + + var qs string + + if req.URL.RawQuery != "" { + qs = fmt.Sprintf("?%s", url.QueryEscape(req.URL.RawQuery)) + } + + uri, err := url.QueryUnescape(strings.Join(parts[1:], "/")) + if err != nil { + if e := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, hostport, assetList, fmt.Sprintf("Error: %s", err), nil, true, "gemini"}); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + if uri != "" { + title = fmt.Sprintf("%s/%s", hostport, uri) + } + + res, err := libgemini.Get( + fmt.Sprintf( + "gemini://%s/%s%s", + hostport, + uri, + qs, + ), + ) + + if err != nil { + if e := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gemini"}); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + if int(res.Header.Status/10) == 3 { + baseURL, err := url.Parse(fmt.Sprintf( + "gemini://%s/%s", + hostport, + uri, + )) + if err != nil { + if e := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error: %s", err), nil, true, "gemini"}); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + http.Redirect(w, req, resolveURI(res.Header.Meta, baseURL), http.StatusFound) + return + } + + if int(res.Header.Status/10) != 2 { + if err := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, fmt.Sprintf("Error %d: %s", res.Header.Status, res.Header.Meta), nil, true, "gemini"}); err != nil { + log.Println("Template error: " + err.Error()) + } + return + } + + if strings.HasPrefix(res.Header.Meta, "text/") { + buf := new(bytes.Buffer) + + _, params, err := mime.ParseMediaType(res.Header.Meta) + if err != nil { + buf.ReadFrom(res.Body) + } else { + encoding, _ := charset.Lookup(params["charset"]) + readbuf := new(bytes.Buffer) + readbuf.ReadFrom(res.Body) + + writer := transform.NewWriter(buf, encoding.NewDecoder()) + writer.Write(readbuf.Bytes()) + writer.Close() + } + + var ( + rawText string + items []Item + ) + + if strings.HasPrefix(res.Header.Meta, libgemini.MIME_GEMINI) { + items = parseGeminiDocument(buf, uri, hostport) + } else { + rawText = buf.String() + } + + if err := tpl.Execute(w, struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string + }{title, fmt.Sprintf("%s/%s", hostport, uri), assetList, rawText, items, false, "gemini"}); err != nil { + log.Println("Template error: " + err.Error()) + } + } else { + io.Copy(w, res.Body) + } + } +} + +// RobotsTxtHandler returns the contents of the robots.txt file +// if configured and valid. +func RobotsTxtHandler(robotstxtdata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if robotstxtdata == nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.Write(robotstxtdata) + } +} + +func FaviconHandler(favicondata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if favicondata == nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "image/vnd.microsoft.icon") + w.Header().Set("Cache-Control", "max-age=2592000") + w.Write(favicondata) + } +} + +func StyleHandler(styledata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/css") + w.Header().Set("Cache-Control", "max-age=2592000") + w.Write(styledata) + } +} + +func JavaScriptHandler(jsdata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/javascript") + w.Header().Set("Cache-Control", "max-age=2592000") + w.Write(jsdata) + } +} + +func FontHandler(woff2 bool, fontdata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if fontdata == nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } + + if woff2 { + w.Header().Set("Content-Type", "font/woff2") + } else { + w.Header().Set("Content-Type", "font/woff") + } + w.Header().Set("Cache-Control", "max-age=2592000") + + w.Write(fontdata) + } +} + +// ListenAndServe creates a listening HTTP server bound to +// the interface specified by bind and sets up a Gopher to HTTP +// proxy proxying requests as requested and by default will prozy +// to a Gopher server address specified by uri if no servers is +// specified by the request. The robots argument is a pointer to +// a robotstxt.RobotsData struct for testing user agents against +// a configurable robots.txt file. +func ListenAndServe(bind, startpagefile string, robotsfile string, robotsdebug bool, vipsconcurrency int) error { + var ( + tpl *template.Template + robotsdata *robotstxt.RobotsData + ) + + robotstxtdata, err := ioutil.ReadFile(robotsfile) + if err != nil { + log.Printf("error reading robots.txt: %s", err) + robotstxtdata = nil + } else { + robotsdata, err = robotstxt.FromBytes(robotstxtdata) + if err != nil { + log.Printf("error reading robots.txt: %s", err) + robotstxtdata = nil + } + } + + box := packr.New("assets", "../assets") + + fontdataw, err := box.Find("iosevka-term-ss03-regular.woff") + if err != nil { + fontdataw = []byte{} + } + fontwAsset := fmt.Sprintf("/iosevka-term-ss03-regular-%x.woff", md5.Sum(fontdataw)) + + fontdataw2, err := box.Find("iosevka-term-ss03-regular.woff2") + if err != nil { + fontdataw2 = []byte{} + } + fontw2Asset := fmt.Sprintf("/iosevka-term-ss03-regular-%x.woff2", md5.Sum(fontdataw2)) + + propfontdataw, err := box.Find("iosevka-aile-regular.woff") + if err != nil { + propfontdataw = []byte{} + } + propfontwAsset := fmt.Sprintf("/iosevka-aile-regular-%x.woff", md5.Sum(propfontdataw)) + + propfontdataw2, err := box.Find("iosevka-aile-regular.woff2") + if err != nil { + propfontdataw2 = []byte{} + } + propfontw2Asset := fmt.Sprintf("/iosevka-aile-regular-%x.woff2", md5.Sum(propfontdataw2)) + + styledata, err := box.Find("style.css") + if err != nil { + styledata = []byte{} + } + styleAsset := fmt.Sprintf("/style-%x.css", md5.Sum(styledata)) + + jsdata, err := box.Find("main.js") + if err != nil { + jsdata = []byte{} + } + jsAsset := fmt.Sprintf("/main-%x.js", md5.Sum(jsdata)) + + favicondata, err := box.Find("favicon.ico") + if err != nil { + favicondata = []byte{} + } + + startpagedata, err := ioutil.ReadFile(startpagefile) + if err != nil { + startpagedata, err = box.Find("startpage.txt") + if err != nil { + startpagedata = []byte{} + } + } + startpagetext := string(startpagedata) + + tpldata, err := ioutil.ReadFile(".template") + if err == nil { + tpltext = string(tpldata) + } + + funcMap := template.FuncMap{ + "safeHtml": func(s string) template.HTML { + return template.HTML(s) + }, + "safeCss": func(s string) template.CSS { + return template.CSS(s) + }, + "safeJs": func(s string) template.JS { + return template.JS(s) + }, + "HTMLEscape": func(s string) string { + return html.EscapeString(s) + }, + "split": strings.Split, + "last": func(s []string) string { + return s[len(s)-1] + }, + "pop": func(s []string) []string { + return s[:len(s)-1] + }, + "replace": func(pattern, output string, input interface{}) string { + var re = regexp.MustCompile(pattern) + var inputStr = fmt.Sprintf("%v", input) + return re.ReplaceAllString(inputStr, output) + }, + "trimLeftChar": func(s string) string { + for i := range s { + if i > 0 { + return s[i:] + } + } + return s[:0] + }, + "hasPrefix": func(s string, prefix string) bool { + return strings.HasPrefix(s, prefix) + }, + "title": func(s string) string { + return strings.Title(s) + }, + } + + tpl, err = template.New("gophermenu").Funcs(funcMap).Parse(tpltext) + if err != nil { + log.Fatal(err) + } + + vips.Startup(&vips.Config{ + ConcurrencyLevel: vipsconcurrency, + }) + + assets := AssetList{styleAsset, jsAsset, fontwAsset, fontw2Asset, propfontwAsset, propfontw2Asset} + + http.Handle("/", gziphandler.GzipHandler(DefaultHandler(tpl, startpagetext, assets))) + http.Handle("/gopher/", gziphandler.GzipHandler(GopherHandler(tpl, robotsdata, assets, robotsdebug))) + http.Handle("/gemini/", gziphandler.GzipHandler(GeminiHandler(tpl, robotsdata, assets, robotsdebug))) + http.Handle("/robots.txt", gziphandler.GzipHandler(RobotsTxtHandler(robotstxtdata))) + http.Handle("/favicon.ico", gziphandler.GzipHandler(FaviconHandler(favicondata))) + http.Handle(styleAsset, gziphandler.GzipHandler(StyleHandler(styledata))) + http.Handle(jsAsset, gziphandler.GzipHandler(JavaScriptHandler(jsdata))) + http.HandleFunc(fontwAsset, FontHandler(false, fontdataw)) + http.HandleFunc(fontw2Asset, FontHandler(true, fontdataw2)) + http.HandleFunc(propfontwAsset, FontHandler(false, propfontdataw)) + http.HandleFunc(propfontw2Asset, FontHandler(true, propfontdataw2)) + //http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("assets/")))) + + return http.ListenAndServe(bind, nil) +} diff --git a/gopherproxy/libgemini/libgemini.go b/gopherproxy/libgemini/libgemini.go new file mode 100644 index 0000000..303490c --- /dev/null +++ b/gopherproxy/libgemini/libgemini.go @@ -0,0 +1,145 @@ +package libgemini + +import ( + "bufio" + "crypto/tls" + "errors" + "fmt" + "io" + "mime" + "net" + "net/url" + "regexp" + "strconv" + "strings" +) + +const ( + CRLF = "\r\n" +) + +const ( + STATUS_INPUT = 10 + STATUS_SUCCESS = 20 + STATUS_SUCCESS_CERT = 21 + STATUS_REDIRECT_TEMP = 30 + STATUS_REDIRECT_PERM = 31 + STATUS_TEMP_FAILURE = 40 + STATUS_SERVER_UNAVAILABLE = 41 + STATUS_CGI_ERROR = 42 + STATUS_PROXY_ERROR = 43 + STATUS_SLOW_DOWN = 44 + STATUS_PERM_FAILURE = 50 + STATUS_NOT_FOUND = 51 + STATUS_GONE = 52 + STATUS_PROXY_REFUSED = 53 + STATUS_BAD_REQUEST = 59 + STATUS_CLIENT_CERT_EXPIRED = 60 + STATUS_TRANSIENT_CERT_REQUEST = 61 + STATUS_AUTH_CERT_REQUIRED = 62 + STATUS_CERT_REJECTED = 63 + STATUS_FUTURE_CERT_REJECTED = 64 + STATUS_EXPIRED_CERT_REJECTED = 65 +) + +const ( + MIME_GEMINI = "text/gemini" + DEFAULT_MIME = MIME_GEMINI + DEFAULT_CHARSET = "utf-8" +) + +var ( + HeaderPattern = regexp.MustCompile("^(\\d\\d)[ \\t]+(.*)$") + LinkPattern = regexp.MustCompile("^=>[ \\t]*([^ \\t]+)(?:[ \\t]+(.*))?$") +) + +type Header struct { + Status int + Meta string +} + +type Response struct { + Header *Header + Body io.Reader +} + +func Get(uri string) (*Response, error) { + u, err := url.Parse(uri) + if err != nil { + return nil, err + } + + if u.Scheme != "gemini" { + return nil, errors.New("invalid scheme for uri") + } + + host := u.Hostname() + port := u.Port() + + if port == "" { + port = "1965" + } + + conn, err := tls.Dial("tcp", net.JoinHostPort(host, port), &tls.Config{ + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: true, + }) + if err != nil { + return nil, err + } + + _, err = conn.Write([]byte(u.String() + CRLF)) + if err != nil { + conn.Close() + return nil, err + } + + reader := bufio.NewReader(conn) + + line, _, err := reader.ReadLine() + if err != nil { + conn.Close() + return nil, err + } + + header, err := ParseHeader(string(line)) + if err != nil { + conn.Close() + return nil, err + } + + return &Response{ + Header: header, + Body: reader, + }, nil +} + +func ParseHeader(line string) (header *Header, err error) { + matches := HeaderPattern.FindStringSubmatch(line) + + status, err := strconv.Atoi(matches[1]) + if err != nil { + return nil, err + } + + meta := matches[2] + + if int(status/10) == 2 { + mediaType, params, err := mime.ParseMediaType(meta) + + if err != nil { + meta = fmt.Sprintf("%s;charset=%s", DEFAULT_MIME, DEFAULT_CHARSET) + } else if strings.HasPrefix(mediaType, "text/") { + if _, ok := params["charset"]; !ok { + meta += ";charset=" + DEFAULT_CHARSET + } + } + } + + header = &Header{ + Status: status, + Meta: meta, + } + + return +} diff --git a/gopherproxy/libgopher/libgopher.go b/gopherproxy/libgopher/libgopher.go new file mode 100644 index 0000000..86d58ff --- /dev/null +++ b/gopherproxy/libgopher/libgopher.go @@ -0,0 +1,312 @@ +package libgopher + +import ( + "bufio" + "errors" + "io" + "log" + "net" + "net/url" + "strings" +) + +// Item Types +const ( + FILE = ItemType('0') // Item is a file + DIRECTORY = ItemType('1') // Item is a directory + PHONEBOOK = ItemType('2') // Item is a CSO phone-book server + ERROR = ItemType('3') // Error + BINHEX = ItemType('4') // Item is a BinHexed Macintosh file. + DOSARCHIVE = ItemType('5') // Item is DOS binary archive of some sort. (*) + UUENCODED = ItemType('6') // Item is a UNIX uuencoded file. + INDEXSEARCH = ItemType('7') // Item is an Index-Search server. + TELNET = ItemType('8') // Item points to a text-based telnet session. + BINARY = ItemType('9') // Item is a binary file! (*) + + // (*) Client must read until the TCP connection is closed. + + REDUNDANT = ItemType('+') // Item is a redundant server + TN3270 = ItemType('T') // Item points to a text-based tn3270 session. + GIF = ItemType('g') // Item is a GIF format graphics file. + IMAGE = ItemType('I') // Item is some kind of image file. + + // non-standard + INFO = ItemType('i') // Item is an informational message + HTML = ItemType('h') // Item is a HTML document + AUDIO = ItemType('s') // Item is an Audio file + PNG = ItemType('p') // Item is a PNG Image + DOC = ItemType('d') // Item is a Document +) + +const ( + // END represents the terminator used in directory responses + END = byte('.') + + // TAB is the delimiter used to separate item response parts + TAB = byte('\t') + + // CRLF is the delimiter used per line of response item + CRLF = "\r\n" + + // DEFAULT is the default item type + DEFAULT = BINARY +) + +// ItemType represents the type of an item +type ItemType byte + +// Return a human friendly represation of an ItemType +func (it ItemType) String() string { + switch it { + case FILE: + return "TXT" + case DIRECTORY: + return "DIR" + case PHONEBOOK: + return "PHO" + case ERROR: + return "ERR" + case BINHEX: + return "HEX" + case DOSARCHIVE: + return "ARC" + case UUENCODED: + return "UUE" + case INDEXSEARCH: + return "QRY" + case TELNET: + return "TEL" + case BINARY: + return "BIN" + case REDUNDANT: + return "DUP" + case TN3270: + return "TN3" + case GIF: + return "GIF" + case IMAGE: + return "IMG" + case INFO: + return "NFO" + case HTML: + return "HTM" + case AUDIO: + return "SND" + case PNG: + return "PNG" + case DOC: + return "DOC" + default: + return "???" + } +} + +// Item describes an entry in a directory listing. +type Item struct { + Type ItemType `json:"type"` + Description string `json:"description"` + Selector string `json:"selector"` + Host string `json:"host"` + Port string `json:"port"` + + // non-standard extensions (ignored by standard clients) + Extras []string `json:"extras"` +} + +// ParseItem parses a line of text into an item +func ParseItem(line string) (item *Item, err error) { + parts := strings.Split(strings.Trim(line, "\r\n"), "\t") + + if len(parts[0]) < 1 { + return nil, errors.New("no item type: " + string(line)) + } + + item = &Item{ + Type: ItemType(parts[0][0]), + Description: string(parts[0][1:]), + Extras: make([]string, 0), + } + + // Selector + if len(parts) > 1 { + item.Selector = string(parts[1]) + } else { + item.Selector = "" + } + + // Host + if len(parts) > 2 { + item.Host = string(parts[2]) + } else { + item.Host = "null.host" + } + + // Port + if len(parts) > 3 { + item.Port = string(parts[3]) + } else { + item.Port = "0" + } + + // Extras + if len(parts) >= 4 { + for _, v := range parts[4:] { + item.Extras = append(item.Extras, string(v)) + } + } + + return +} + +func (i *Item) isDirectoryLike() bool { + switch i.Type { + case DIRECTORY: + return true + case INDEXSEARCH: + return true + default: + return false + } +} + +// Directory representes a Gopher Menu of Items +type Directory struct { + Items []*Item `json:"items"` +} + +// Response represents a Gopher resource that +// Items contains a non-empty array of Item(s) +// for directory types, otherwise the Body +// contains the fetched resource (file, image, etc). +type Response struct { + Type ItemType + Dir Directory + Body io.Reader +} + +// Get fetches a Gopher resource by URI +func Get(uri string) (*Response, error) { + u, err := url.Parse(uri) + if err != nil { + return nil, err + } + + if u.Scheme != "gopher" { + return nil, errors.New("invalid scheme for uri") + } + + host := u.Hostname() + port := u.Port() + + if port == "" { + port = "70" + } + + var ( + Type ItemType + Selector string + ) + + path := strings.TrimPrefix(u.Path, "/") + if len(path) > 2 { + Type = ItemType(path[0]) + Selector = path[1:] + if u.RawQuery != "" { + Selector += "\t" + u.RawQuery + } + } else if len(path) == 1 { + Type = ItemType(path[0]) + Selector = "" + } else { + Type = ItemType(DIRECTORY) + Selector = "" + } + + i := Item{Type: Type, Selector: Selector, Host: host, Port: port} + res := Response{Type: i.Type} + + if i.isDirectoryLike() { + d, err := i.FetchDirectory() + if err != nil { + return nil, err + } + + res.Dir = d + } else { + reader, err := i.FetchFile() + if err != nil { + return nil, err + } + + res.Body = reader + } + + return &res, nil +} + +// FetchFile fetches data, not directory information. +// Calling this on a DIRECTORY Item type +// or unsupported type will return an error. +func (i *Item) FetchFile() (io.Reader, error) { + if i.Type == DIRECTORY { + return nil, errors.New("cannot fetch a directory as a file") + } + + conn, err := net.Dial("tcp", net.JoinHostPort(i.Host, i.Port)) + if err != nil { + return nil, err + } + + _, err = conn.Write([]byte(i.Selector + CRLF)) + if err != nil { + conn.Close() + return nil, err + } + + return conn, nil +} + +// FetchDirectory fetches directory information, not data. +// Calling this on an Item whose type is not DIRECTORY will return an error. +func (i *Item) FetchDirectory() (Directory, error) { + if !i.isDirectoryLike() { + return Directory{}, errors.New("cannot fetch a file as a directory") + } + + conn, err := net.Dial("tcp", net.JoinHostPort(i.Host, i.Port)) + if err != nil { + return Directory{}, err + } + + _, err = conn.Write([]byte(i.Selector + CRLF)) + if err != nil { + return Directory{}, err + } + + reader := bufio.NewReader(conn) + scanner := bufio.NewScanner(reader) + scanner.Split(bufio.ScanLines) + + var items []*Item + + for scanner.Scan() { + line := strings.Trim(scanner.Text(), "\r\n") + + if len(line) == 0 { + continue + } + + if len(line) == 1 && line[0] == END { + break + } + + item, err := ParseItem(line) + if err != nil { + log.Printf("Error parsing %q: %q", line, err) + continue + } + items = append(items, item) + } + + return Directory{items}, nil +} diff --git a/gopherproxy/template.go b/gopherproxy/template.go new file mode 100644 index 0000000..6b90cc0 --- /dev/null +++ b/gopherproxy/template.go @@ -0,0 +1,122 @@ +package gopherproxy + +var tpltext = ` + + + + + {{ .Title }}{{ if ne .Protocol "startpage" }} - {{ .Protocol | title }} proxy{{ end }} + + + + +
+
+ {{ .Protocol }}://:// + + {{- if .URI -}} + {{- $page := . -}} + {{- $href := printf "/%s" .Protocol -}} + {{- $uriParts := split .URI "/" -}} + + {{- $uriLast := $uriParts | last -}} + {{- $uriParts = $uriParts | pop -}} + {{- if eq $uriLast "" -}} + {{- $uriLast = $uriParts | last -}} + {{- $uriParts = $uriParts | pop -}} + {{- end -}} + + {{- range $i, $part := $uriParts -}} + {{- if and (eq $page.Protocol "gopher") (eq $i 1) -}} + {{- $href = printf "%s/1" $href -}} + {{- $part = $part | trimLeftChar -}} + {{- if not (eq $part "") -}} + {{- $href = printf "%s/%s" $href $part -}} + /{{ $part }} + {{- end -}} + {{- else -}} + {{- $href = printf "%s/%s" $href . -}} + {{- if ne $i 0 -}} + / + {{- end -}} + {{ . }} + {{- end -}} + {{- end -}} + {{- if ne (len $uriParts) 0 -}} + / + {{- end -}} + {{- if and (eq $page.Protocol "gopher") (eq (len $uriParts) 1) -}} + {{- $uriLast = $uriLast | trimLeftChar -}} + {{- end -}} + {{ $uriLast }} + {{- end -}} +
+
+ {{- if and (not .Lines) (not .Error) (eq .Protocol "gopher") -}} +
View raw
+ {{- end -}} +
+
+
+
+ 
+				{{- if .Lines -}}
+					{{- $content := "" -}}
+					{{- range .Lines -}}
+						{{- if ne $content "" -}}
+							{{- $content = printf "%s\n" $content -}}
+						{{- end -}}
+						{{- if .Link -}}
+							{{- $content = printf "%s%s" $content (printf "%s  %s" .Type .Type .Link (.Text | HTMLEscape)) -}}
+						{{- else -}}
+							{{- $content = printf "%s%s" $content (printf "     %s" (.Text | HTMLEscape)) -}}
+						{{- end -}}
+					{{- end -}}
+					{{- $content | safeHtml -}}
+				{{- else -}}
+					{{- .RawText -}}
+				{{- end -}}
+
+
+ + + +` diff --git a/libgemini.go b/libgemini.go deleted file mode 100644 index 94744ba..0000000 --- a/libgemini.go +++ /dev/null @@ -1,153 +0,0 @@ -package gopherproxy - -import ( - "bufio" - "crypto/tls" - "errors" - "io" - "mime" - "net/url" - "regexp" - "strconv" - "strings" -) - -const ( - CRLF = "\r\n" -) - -const ( - STATUS_INPUT = 10 - STATUS_SUCCESS = 20 - STATUS_SUCCESS_CERT = 21 - STATUS_REDIRECT_TEMP = 30 - STATUS_REDIRECT_PERM = 31 - STATUS_TEMP_FAILURE = 40 - STATUS_SERVER_UNAVAILABLE = 41 - STATUS_CGI_ERROR = 42 - STATUS_PROXY_ERROR = 43 - STATUS_SLOW_DOWN = 44 - STATUS_PERM_FAILURE = 50 - STATUS_NOT_FOUND = 51 - STATUS_GONE = 52 - STATUS_PROXY_REFUSED = 53 - STATUS_BAD_REQUEST = 59 - STATUS_CLIENT_CERT_EXPIRED = 60 - STATUS_TRANSIENT_CERT_REQUEST = 61 - STATUS_AUTH_CERT_REQUIRED = 62 - STATUS_CERT_REJECTED = 63 - STATUS_FUTURE_CERT_REJECTED = 64 - STATUS_EXPIRED_CERT_REJECTED = 65 -) - -const ( - MIME_GEMINI = "text/gemini" - DEFAULT_MIME = MIME_GEMINI - DEFAULT_CHARSET = "utf-8" -) - -var ( - HeaderPattern = regexp.MustCompile("^(\\d\\d)[ \\t]+(.*)$") - GeminiLinkPattern = regexp.MustCompile("^=>[ \\t]*([^ \\t]+)(?:[ \\t]+(.*))?$") -) - -type GeminiHeader struct { - Status int - Meta string -} - -type GeminiResponse struct { - Header *GeminiHeader - Body io.Reader -} - -func GeminiGet(uri string) (*GeminiResponse, error) { - u, err := url.Parse(uri) - if err != nil { - return nil, err - } - - if u.Scheme != "gemini" { - return nil, errors.New("invalid scheme for uri") - } - - var ( - host string - port int - ) - - hostport := strings.Split(u.Host, ":") - if len(hostport) == 2 { - host = hostport[0] - n, err := strconv.ParseInt(hostport[1], 10, 32) - if err != nil { - return nil, err - } - port = int(n) - } else { - host, port = hostport[0], 1965 - } - - conn, err := tls.Dial("tcp", host+":"+strconv.Itoa(port), &tls.Config{ - MinVersion: tls.VersionTLS12, - InsecureSkipVerify: true, - }) - if err != nil { - return nil, err - } - - _, err = conn.Write([]byte(u.String() + CRLF)) - if err != nil { - conn.Close() - return nil, err - } - - reader := bufio.NewReader(conn) - - line, _, err := reader.ReadLine() - if err != nil { - conn.Close() - return nil, err - } - - header, err := ParseGeminiHeader(string(line)) - if err != nil { - conn.Close() - return nil, err - } - - return &GeminiResponse{ - Header: header, - Body: reader, - }, nil -} - -func ParseGeminiHeader(line string) (header *GeminiHeader, err error) { - matches := HeaderPattern.FindStringSubmatch(line) - - status, err := strconv.Atoi(matches[1]) - if err != nil { - return nil, err - } - - meta := matches[2] - - if int(status/10) == 2 { - mediaType, params, err := mime.ParseMediaType(meta) - - if err != nil { - meta = DEFAULT_MIME + ";charset=" + DEFAULT_CHARSET - } else if strings.HasPrefix(mediaType, "text/") { - if _, ok := params["charset"]; !ok { - meta += ";charset=" + DEFAULT_CHARSET - } - } - } - - header = &GeminiHeader{ - Status: status, - Meta: meta, - } - - return -} diff --git a/main.go b/main.go new file mode 100644 index 0000000..f5a82fe --- /dev/null +++ b/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "flag" + "log" + + "git.feuerfuchs.dev/Feuerfuchs/gopherproxy/gopherproxy" +) + +var ( + // TODO: Allow config file and environment vars + // (opt -> env -> config -> default) + bind = flag.String("bind", "0.0.0.0:8000", "[int]:port to bind to") + startpagefile = flag.String("startpage-file", "startpage.txt", "Default page to display if no URL is specified") + robotsfile = flag.String("robots-file", "robots.txt", "robots.txt file") + robotsdebug = flag.Bool("robots-debug", false, "print output about ignored robots.txt") + vipsconcurrency = flag.Int("vips-concurrency", 1, "Concurrency level of libvips") +) + +func main() { + flag.Parse() + + // Use a config struct + log.Fatal(gopherproxy.ListenAndServe(*bind, *startpagefile, *robotsfile, *robotsdebug, *vipsconcurrency)) +} diff --git a/template.go b/template.go deleted file mode 100644 index 6b90cc0..0000000 --- a/template.go +++ /dev/null @@ -1,122 +0,0 @@ -package gopherproxy - -var tpltext = ` - - - - - {{ .Title }}{{ if ne .Protocol "startpage" }} - {{ .Protocol | title }} proxy{{ end }} - - - - -
-
- {{ .Protocol }}://:// - - {{- if .URI -}} - {{- $page := . -}} - {{- $href := printf "/%s" .Protocol -}} - {{- $uriParts := split .URI "/" -}} - - {{- $uriLast := $uriParts | last -}} - {{- $uriParts = $uriParts | pop -}} - {{- if eq $uriLast "" -}} - {{- $uriLast = $uriParts | last -}} - {{- $uriParts = $uriParts | pop -}} - {{- end -}} - - {{- range $i, $part := $uriParts -}} - {{- if and (eq $page.Protocol "gopher") (eq $i 1) -}} - {{- $href = printf "%s/1" $href -}} - {{- $part = $part | trimLeftChar -}} - {{- if not (eq $part "") -}} - {{- $href = printf "%s/%s" $href $part -}} - /{{ $part }} - {{- end -}} - {{- else -}} - {{- $href = printf "%s/%s" $href . -}} - {{- if ne $i 0 -}} - / - {{- end -}} - {{ . }} - {{- end -}} - {{- end -}} - {{- if ne (len $uriParts) 0 -}} - / - {{- end -}} - {{- if and (eq $page.Protocol "gopher") (eq (len $uriParts) 1) -}} - {{- $uriLast = $uriLast | trimLeftChar -}} - {{- end -}} - {{ $uriLast }} - {{- end -}} -
-
- {{- if and (not .Lines) (not .Error) (eq .Protocol "gopher") -}} -
View raw
- {{- end -}} -
-
-
-
- 
-				{{- if .Lines -}}
-					{{- $content := "" -}}
-					{{- range .Lines -}}
-						{{- if ne $content "" -}}
-							{{- $content = printf "%s\n" $content -}}
-						{{- end -}}
-						{{- if .Link -}}
-							{{- $content = printf "%s%s" $content (printf "%s  %s" .Type .Type .Link (.Text | HTMLEscape)) -}}
-						{{- else -}}
-							{{- $content = printf "%s%s" $content (printf "     %s" (.Text | HTMLEscape)) -}}
-						{{- end -}}
-					{{- end -}}
-					{{- $content | safeHtml -}}
-				{{- else -}}
-					{{- .RawText -}}
-				{{- end -}}
-
-
- - - -` -- cgit v1.2.3-70-g09d2