From 4bf44b16562335b3d09b6df0150521bb5b5f776f Mon Sep 17 00:00:00 2001 From: Feuerfuchs Date: Mon, 18 May 2020 12:12:43 +0200 Subject: WIP: Refactoring --- internal/port/gemini.go | 205 ++++++++++++++++++++++++++ internal/port/gopher.go | 217 ++++++++++++++++++++++++++++ internal/port/main.go | 301 +++++++++++++++++++++++++++++++++++++++ internal/port/tpl/gemini.html | 0 internal/port/tpl/gopher.html | 0 internal/port/tpl/startpage.html | 120 ++++++++++++++++ 6 files changed, 843 insertions(+) create mode 100644 internal/port/gemini.go create mode 100644 internal/port/gopher.go create mode 100644 internal/port/main.go create mode 100644 internal/port/tpl/gemini.html create mode 100644 internal/port/tpl/gopher.html create mode 100644 internal/port/tpl/startpage.html (limited to 'internal') diff --git a/internal/port/gemini.go b/internal/port/gemini.go new file mode 100644 index 0000000..f9b0b97 --- /dev/null +++ b/internal/port/gemini.go @@ -0,0 +1,205 @@ +package port + +import ( + "bytes" + "fmt" + "html/template" + "io" + "log" + "mime" + "net/http" + "net/url" + "regexp" + "strings" + + "golang.org/x/net/html/charset" + "golang.org/x/text/transform" + + "git.vulpes.one/Feuerfuchs/port/port/libgemini" + + "github.com/temoto/robotstxt" +) + +var ( + TermEscapeSGRPattern = regexp.MustCompile("\\[\\d+(;\\d+)*m") +) + +func resolveURI(uri string, baseURL *url.URL) (resolvedURI string) { + if strings.HasPrefix(uri, "//") { + resolvedURI = "/gemini/" + strings.TrimPrefix(uri, "//") + } else if strings.HasPrefix(uri, "gemini://") { + resolvedURI = "/gemini/" + strings.TrimPrefix(uri, "gemini://") + } else if strings.HasPrefix(uri, "gopher://") { + resolvedURI = "/gopher/" + strings.TrimPrefix(uri, "gopher://") + } else { + url, err := url.Parse(uri) + if err != nil { + return "" + } + adjustedURI := baseURL.ResolveReference(url) + path := adjustedURI.Path + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if adjustedURI.Scheme == "gemini" { + resolvedURI = "/gemini/" + adjustedURI.Host + path + } else if adjustedURI.Scheme == "gopher" { + resolvedURI = "/gopher/" + adjustedURI.Host + path + } else { + resolvedURI = adjustedURI.String() + } + } + + return +} + +func GeminiHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, assetList AssetList, robotsdebug bool) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + agent := req.UserAgent() + path := strings.TrimPrefix(req.URL.Path, "/gemini/") + + if robotsdata != nil && robotsdebug && !robotsdata.TestAgent(path, agent) { + log.Printf("UserAgent %s ignored robots.txt", agent) + } + + parts := strings.Split(path, "/") + hostport := parts[0] + + if len(hostport) == 0 { + http.Redirect(w, req, "/", http.StatusFound) + return + } + + title := hostport + + var qs string + + if req.URL.RawQuery != "" { + qs = fmt.Sprintf("?%s", url.QueryEscape(req.URL.RawQuery)) + } + + uri, err := url.QueryUnescape(strings.Join(parts[1:], "/")) + if err != nil { + if e := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: hostport, + Assets: assetList, + RawText: fmt.Sprintf("Error: %s", err), + Error: true, + Protocol: "gemini", + }); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + if uri != "" { + title = fmt.Sprintf("%s/%s", hostport, uri) + } + + res, err := libgemini.Get( + fmt.Sprintf( + "gemini://%s/%s%s", + hostport, + uri, + qs, + ), + ) + + if err != nil { + if e := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + RawText: fmt.Sprintf("Error: %s", err), + Error: true, + Protocol: "gemini", + }); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + if int(res.Header.Status/10) == 3 { + baseURL, err := url.Parse(fmt.Sprintf( + "gemini://%s/%s", + hostport, + uri, + )) + if err != nil { + if e := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + RawText: fmt.Sprintf("Error: %s", err), + Error: true, + Protocol: "gemini", + }); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + http.Redirect(w, req, resolveURI(res.Header.Meta, baseURL), http.StatusFound) + return + } + + if int(res.Header.Status/10) != 2 { + if err := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + RawText: fmt.Sprintf("Error %d: %s", res.Header.Status, res.Header.Meta), + Error: true, + Protocol: "gemini", + }); err != nil { + log.Println("Template error: " + err.Error()) + } + return + } + + if strings.HasPrefix(res.Header.Meta, "text/") { + buf := new(bytes.Buffer) + + _, params, err := mime.ParseMediaType(res.Header.Meta) + if err != nil { + buf.ReadFrom(res.Body) + } else { + encoding, _ := charset.Lookup(params["charset"]) + readbuf := new(bytes.Buffer) + readbuf.ReadFrom(res.Body) + + writer := transform.NewWriter(buf, encoding.NewDecoder()) + writer.Write(readbuf.Bytes()) + writer.Close() + } + + var ( + rawText string + items []Item + ) + + if strings.HasPrefix(res.Header.Meta, libgemini.MIME_GEMINI) { + items = parseGeminiDocument(buf, uri, hostport) + } else { + rawText = buf.String() + } + + if err := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + Lines: items, + RawText: rawText, + Protocol: "gemini", + }); err != nil { + log.Println("Template error: " + err.Error()) + } + } else { + io.Copy(w, res.Body) + } + } +} diff --git a/internal/port/gopher.go b/internal/port/gopher.go new file mode 100644 index 0000000..ebeb213 --- /dev/null +++ b/internal/port/gopher.go @@ -0,0 +1,217 @@ +package port + +import ( + "bytes" + "fmt" + "html/template" + "io" + "log" + "net" + "net/http" + "net/url" + "strings" + + "git.vulpes.one/Feuerfuchs/port/port/libgopher" + + "github.com/davidbyttow/govips/pkg/vips" + "github.com/temoto/robotstxt" +) + +type Item struct { + Link template.URL + Type string + Text string +} + +func renderGopherDirectory(w http.ResponseWriter, tpl *template.Template, assetList AssetList, uri string, hostport string, d libgopher.Directory) error { + var title string + + out := make([]Item, len(d.Items)) + + for i, x := range d.Items { + if x.Type == libgopher.INFO && x.Selector == "TITLE" { + title = x.Description + continue + } + + tr := Item{ + Text: x.Description, + Type: x.Type.String(), + } + + if x.Type == libgopher.INFO { + out[i] = tr + continue + } + + if strings.HasPrefix(x.Selector, "URL:") || strings.HasPrefix(x.Selector, "/URL:") { + link := strings.TrimPrefix(strings.TrimPrefix(x.Selector, "/"), "URL:") + if strings.HasPrefix(link, "gemini://") { + link = fmt.Sprintf( + "/gemini/%s", + strings.TrimPrefix(link, "gemini://"), + ) + } else if strings.HasPrefix(link, "gopher://") { + link = fmt.Sprintf( + "/gopher/%s", + strings.TrimPrefix(link, "gopher://"), + ) + } + tr.Link = template.URL(link) + } else { + var linkHostport string + if x.Port != "70" { + linkHostport = net.JoinHostPort(x.Host, x.Port) + } else { + linkHostport = x.Host + } + + path := url.PathEscape(x.Selector) + path = strings.Replace(path, "%2F", "/", -1) + tr.Link = template.URL( + fmt.Sprintf( + "/gopher/%s/%s%s", + linkHostport, + string(byte(x.Type)), + path, + ), + ) + } + + out[i] = tr + } + + if title == "" { + if uri != "" { + title = fmt.Sprintf("%s/%s", hostport, uri) + } else { + title = hostport + } + } + + return tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + Lines: out, + Protocol: "gopher", + }) +} + +// GopherHandler returns a Handler that proxies requests +// to the specified Gopher server as denoated by the first argument +// to the request path and renders the content using the provided template. +// The optional robots parameters points to a robotstxt.RobotsData struct +// to test user agents against a configurable robotst.txt file. +func GopherHandler(tpl *template.Template, robotsdata *robotstxt.RobotsData, assetList AssetList, robotsdebug bool) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + agent := req.UserAgent() + path := strings.TrimPrefix(req.URL.Path, "/gopher/") + + if robotsdata != nil && robotsdebug && !robotsdata.TestAgent(path, agent) { + log.Printf("UserAgent %s ignored robots.txt", agent) + } + + parts := strings.Split(path, "/") + hostport := parts[0] + + if len(hostport) == 0 { + http.Redirect(w, req, "/", http.StatusFound) + return + } + + title := hostport + + var qs string + + if req.URL.RawQuery != "" { + qs = fmt.Sprintf("?%s", url.QueryEscape(req.URL.RawQuery)) + } + + uri, err := url.QueryUnescape(strings.Join(parts[1:], "/")) + if err != nil { + if e := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: hostport, + Assets: assetList, + RawText: fmt.Sprintf("Error: %s", err), + Error: true, + Protocol: "gopher", + }); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(err.Error()) + } + return + } + + if uri != "" { + title = fmt.Sprintf("%s/%s", hostport, uri) + } + + res, err := libgopher.Get( + fmt.Sprintf( + "gopher://%s/%s%s", + hostport, + uri, + qs, + ), + ) + + if err != nil { + if e := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + RawText: fmt.Sprintf("Error: %s", err), + Error: true, + Protocol: "gopher", + }); e != nil { + log.Println("Template error: " + e.Error()) + } + return + } + + if res.Body != nil { + if len(parts) < 2 { + io.Copy(w, res.Body) + } else if strings.HasPrefix(parts[1], "0") && !strings.HasSuffix(uri, ".xml") && !strings.HasSuffix(uri, ".asc") { + buf := new(bytes.Buffer) + buf.ReadFrom(res.Body) + + if err := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + RawText: buf.String(), + Protocol: "gopher", + }); err != nil { + log.Println("Template error: " + err.Error()) + } + } else if strings.HasPrefix(parts[1], "T") { + _, _, err = vips.NewTransform(). + Load(res.Body). + ResizeStrategy(vips.ResizeStrategyAuto). + ResizeWidth(160). + Quality(75). + Output(w). + Apply() + } else { + io.Copy(w, res.Body) + } + } else { + if err := renderGopherDirectory(w, tpl, assetList, uri, hostport, res.Dir); err != nil { + if e := tpl.Execute(w, TemplateVariables{ + Title: title, + URI: fmt.Sprintf("%s/%s", hostport, uri), + Assets: assetList, + RawText: fmt.Sprintf("Error: %s", err), + Error: true, + Protocol: "gopher", + }); e != nil { + log.Println("Template error: " + e.Error()) + log.Println(e.Error()) + } + } + } + } +} diff --git a/internal/port/main.go b/internal/port/main.go new file mode 100644 index 0000000..5cdd794 --- /dev/null +++ b/internal/port/main.go @@ -0,0 +1,301 @@ +package port + +import ( + "crypto/md5" + "fmt" + "html" + "html/template" + "io/ioutil" + "log" + "net/http" + "regexp" + "strings" + + "github.com/NYTimes/gziphandler" + "github.com/davidbyttow/govips/pkg/vips" + "github.com/gobuffalo/packr/v2" + "github.com/temoto/robotstxt" +) + +type AssetList struct { + Style string + JS string + FontW string + FontW2 string + PropFontW string + PropFontW2 string +} + +type TemplateVariables struct { + Title string + URI string + Assets AssetList + RawText string + Lines []Item + Error bool + Protocol string +} + +func DefaultHandler(tpl *template.Template, startpagetext string, assetList AssetList) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if err := tpl.Execute(w, TemplateVariables{ + Title: "Gopher/Gemini proxy", + Assets: assetList, + RawText: startpagetext, + Protocol: "startpage", + }); err != nil { + log.Println("Template error: " + err.Error()) + } + } +} + +// RobotsTxtHandler returns the contents of the robots.txt file +// if configured and valid. +func RobotsTxtHandler(robotstxtdata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if robotstxtdata == nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.Write(robotstxtdata) + } +} + +func FaviconHandler(favicondata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if favicondata == nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "image/vnd.microsoft.icon") + w.Header().Set("Cache-Control", "max-age=2592000") + w.Write(favicondata) + } +} + +func StyleHandler(styledata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/css") + w.Header().Set("Cache-Control", "max-age=2592000") + w.Write(styledata) + } +} + +func JavaScriptHandler(jsdata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/javascript") + w.Header().Set("Cache-Control", "max-age=2592000") + w.Write(jsdata) + } +} + +func FontHandler(woff2 bool, fontdata []byte) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if fontdata == nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } + + if woff2 { + w.Header().Set("Content-Type", "font/woff2") + } else { + w.Header().Set("Content-Type", "font/woff") + } + w.Header().Set("Cache-Control", "max-age=2592000") + + w.Write(fontdata) + } +} + +// ListenAndServe creates a listening HTTP server bound to +// the interface specified by bind and sets up a Gopher to HTTP +// proxy proxying requests as requested and by default will prozy +// to a Gopher server address specified by uri if no servers is +// specified by the request. The robots argument is a pointer to +// a robotstxt.RobotsData struct for testing user agents against +// a configurable robots.txt file. +func ListenAndServe(bind, startpagefile string, robotsfile string, robotsdebug bool, vipsconcurrency int) error { + box := packr.New("assets", "../assets") + + // + // Robots + + var robotsdata *robotstxt.RobotsData + + robotstxtdata, err := ioutil.ReadFile(robotsfile) + if err != nil { + log.Printf("error reading robots.txt: %s", err) + robotstxtdata = nil + } else { + robotsdata, err = robotstxt.FromBytes(robotstxtdata) + if err != nil { + log.Printf("error reading robots.txt: %s", err) + robotstxtdata = nil + } + } + + // + // Fonts + + fontdataw, err := box.Find("iosevka-term-ss03-regular.woff") + if err != nil { + fontdataw = []byte{} + } + fontwAsset := fmt.Sprintf("/iosevka-term-ss03-regular-%x.woff", md5.Sum(fontdataw)) + + fontdataw2, err := box.Find("iosevka-term-ss03-regular.woff2") + if err != nil { + fontdataw2 = []byte{} + } + fontw2Asset := fmt.Sprintf("/iosevka-term-ss03-regular-%x.woff2", md5.Sum(fontdataw2)) + + propfontdataw, err := box.Find("iosevka-aile-regular.woff") + if err != nil { + propfontdataw = []byte{} + } + propfontwAsset := fmt.Sprintf("/iosevka-aile-regular-%x.woff", md5.Sum(propfontdataw)) + + propfontdataw2, err := box.Find("iosevka-aile-regular.woff2") + if err != nil { + propfontdataw2 = []byte{} + } + propfontw2Asset := fmt.Sprintf("/iosevka-aile-regular-%x.woff2", md5.Sum(propfontdataw2)) + + // + // Stylesheet + + styledata, err := box.Find("style.css") + if err != nil { + styledata = []byte{} + } + styleAsset := fmt.Sprintf("/style-%x.css", md5.Sum(styledata)) + + // + // JavaScript + + jsdata, err := box.Find("main.js") + if err != nil { + jsdata = []byte{} + } + jsAsset := fmt.Sprintf("/main-%x.js", md5.Sum(jsdata)) + + // + // Favicon + + favicondata, err := box.Find("favicon.ico") + if err != nil { + favicondata = []byte{} + } + + // + // Start page text + + startpagedata, err := ioutil.ReadFile(startpagefile) + if err != nil { + startpagedata, err = box.Find("startpage.txt") + if err != nil { + startpagedata = []byte{} + } + } + startpagetext := string(startpagedata) + + // + // + + var allFiles []string + files, err := ioutil.ReadDir("./tpl") + if err != nil { + fmt.Println(err) + } + for _, file := range files { + filename := file.Name() + if strings.HasSuffix(filename, ".html") { + allFiles = append(allFiles, "./tpl/"+filename) + } + } + + templates, err = template.ParseFiles(allFiles...) + + // + + funcMap := template.FuncMap{ + "safeHtml": func(s string) template.HTML { + return template.HTML(s) + }, + "safeCss": func(s string) template.CSS { + return template.CSS(s) + }, + "safeJs": func(s string) template.JS { + return template.JS(s) + }, + "HTMLEscape": func(s string) string { + return html.EscapeString(s) + }, + "split": strings.Split, + "last": func(s []string) string { + return s[len(s)-1] + }, + "pop": func(s []string) []string { + return s[:len(s)-1] + }, + "replace": func(pattern, output string, input interface{}) string { + var re = regexp.MustCompile(pattern) + var inputStr = fmt.Sprintf("%v", input) + return re.ReplaceAllString(inputStr, output) + }, + "trimLeftChar": func(s string) string { + for i := range s { + if i > 0 { + return s[i:] + } + } + return s[:0] + }, + "hasPrefix": func(s string, prefix string) bool { + return strings.HasPrefix(s, prefix) + }, + "title": func(s string) string { + return strings.Title(s) + }, + } + + // + + startpageTpl := templates.Lookup("startpage.html").Funcs(funcMap) + geminiTpl := templates.Lookup("gemini.html").Funcs(funcMap) + gopherTpl := templates.Lookup("gopher.html").Funcs(funcMap) + + // + // + + vips.Startup(&vips.Config{ + ConcurrencyLevel: vipsconcurrency, + }) + + assets := AssetList{ + Style: styleAsset, + JS: jsAsset, + FontW: fontwAsset, + FontW2: fontw2Asset, + PropFontW: propfontwAsset, + PropFontW2: propfontw2Asset, + } + + http.Handle("/", gziphandler.GzipHandler(DefaultHandler(startpageTpl, startpagetext, assets))) + http.Handle("/gopher/", gziphandler.GzipHandler(GopherHandler(gopherTpl, robotsdata, assets, robotsdebug))) + http.Handle("/gemini/", gziphandler.GzipHandler(GeminiHandler(geminiTpl, robotsdata, assets, robotsdebug))) + http.Handle("/robots.txt", gziphandler.GzipHandler(RobotsTxtHandler(robotstxtdata))) + http.Handle("/favicon.ico", gziphandler.GzipHandler(FaviconHandler(favicondata))) + http.Handle(styleAsset, gziphandler.GzipHandler(StyleHandler(styledata))) + http.Handle(jsAsset, gziphandler.GzipHandler(JavaScriptHandler(jsdata))) + http.HandleFunc(fontwAsset, FontHandler(false, fontdataw)) + http.HandleFunc(fontw2Asset, FontHandler(true, fontdataw2)) + http.HandleFunc(propfontwAsset, FontHandler(false, propfontdataw)) + http.HandleFunc(propfontw2Asset, FontHandler(true, propfontdataw2)) + //http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("assets/")))) + + return http.ListenAndServe(bind, nil) +} diff --git a/internal/port/tpl/gemini.html b/internal/port/tpl/gemini.html new file mode 100644 index 0000000..e69de29 diff --git a/internal/port/tpl/gopher.html b/internal/port/tpl/gopher.html new file mode 100644 index 0000000..e69de29 diff --git a/internal/port/tpl/startpage.html b/internal/port/tpl/startpage.html new file mode 100644 index 0000000..8482a6f --- /dev/null +++ b/internal/port/tpl/startpage.html @@ -0,0 +1,120 @@ + + + + + + {{ .Title }}{{ if ne .Protocol "startpage" }} - {{ .Protocol | title }} proxy{{ end }} + + + + +
+
+ {{ .Protocol }}://:// + + {{- if .URI -}} + {{- $page := . -}} + {{- $href := printf "/%s" .Protocol -}} + {{- $uriParts := split .URI "/" -}} + + {{- $uriLast := $uriParts | last -}} + {{- $uriParts = $uriParts | pop -}} + {{- if eq $uriLast "" -}} + {{- $uriLast = $uriParts | last -}} + {{- $uriParts = $uriParts | pop -}} + {{- end -}} + + {{- range $i, $part := $uriParts -}} + {{- if and (eq $page.Protocol "gopher") (eq $i 1) -}} + {{- $href = printf "%s/1" $href -}} + {{- $part = $part | trimLeftChar -}} + {{- if not (eq $part "") -}} + {{- $href = printf "%s/%s" $href $part -}} + /{{ $part }} + {{- end -}} + {{- else -}} + {{- $href = printf "%s/%s" $href . -}} + {{- if ne $i 0 -}} + / + {{- end -}} + {{ . }} + {{- end -}} + {{- end -}} + {{- if ne (len $uriParts) 0 -}} + / + {{- end -}} + {{- if and (eq $page.Protocol "gopher") (eq (len $uriParts) 1) -}} + {{- $uriLast = $uriLast | trimLeftChar -}} + {{- end -}} + {{ $uriLast }} + {{- end -}} +
+
+ {{- if and (not .Lines) (not .Error) (eq .Protocol "gopher") -}} +
View raw
+ {{- end -}} +
+
+
+
+ 
+				{{- if .Lines -}}
+					{{- $content := "" -}}
+					{{- range .Lines -}}
+						{{- if ne $content "" -}}
+							{{- $content = printf "%s\n" $content -}}
+						{{- end -}}
+						{{- if .Link -}}
+							{{- $content = printf "%s%s" $content (printf "%s  %s" .Type .Type .Link (.Text | HTMLEscape)) -}}
+						{{- else -}}
+							{{- $content = printf "%s%s" $content (printf "     %s" (.Text | HTMLEscape)) -}}
+						{{- end -}}
+					{{- end -}}
+					{{- $content | safeHtml -}}
+				{{- else -}}
+					{{- .RawText -}}
+				{{- end -}}
+
+
+ + + + -- cgit v1.2.3-70-g09d2