From 3df3405ad537a3410e468b9d9eabd143b44d2f5d Mon Sep 17 00:00:00 2001
From: Steven <stevenlgtm@gmail.com>
Date: Tue, 20 Aug 2024 23:17:24 +0800
Subject: [PATCH] chore: tweak plugin
---
internal/mailer/html2text.go | 103 ---------------
internal/mailer/mailer.go | 43 ------
internal/mailer/sendmail.go | 78 -----------
internal/mailer/smtp.go | 196 ----------------------------
internal/mailer/smtp_test.go | 164 -----------------------
plugin/httpgetter/html_meta.go | 98 ++++++++++++++
plugin/httpgetter/html_meta_test.go | 19 +++
plugin/httpgetter/http_getter.go | 4 +
plugin/httpgetter/image.go | 45 +++++++
plugin/httpgetter/util.go | 15 +++
10 files changed, 181 insertions(+), 584 deletions(-)
delete mode 100644 internal/mailer/html2text.go
delete mode 100644 internal/mailer/mailer.go
delete mode 100644 internal/mailer/sendmail.go
delete mode 100644 internal/mailer/smtp.go
delete mode 100644 internal/mailer/smtp_test.go
create mode 100644 plugin/httpgetter/html_meta.go
create mode 100644 plugin/httpgetter/html_meta_test.go
create mode 100644 plugin/httpgetter/http_getter.go
create mode 100644 plugin/httpgetter/image.go
create mode 100644 plugin/httpgetter/util.go
diff --git a/internal/mailer/html2text.go b/internal/mailer/html2text.go
deleted file mode 100644
index 0bf07a1..0000000
--- a/internal/mailer/html2text.go
+++ /dev/null
@@ -1,103 +0,0 @@
-package mailer
-
-import (
- "regexp"
- "strings"
-
- "golang.org/x/net/html"
-)
-
-var whitespaceRegex = regexp.MustCompile(`\s+`)
-
-// Very rudimentary auto HTML to Text mail body converter.
-//
-// Caveats:
-// - This method doesn't check for correctness of the HTML document.
-// - Links will be converted to "[text](url)" format.
-// - List items (<li>) are prefixed with "- ".
-// - Indentation is stripped (both tabs and spaces).
-// - Trailing spaces are preserved.
-// - Multiple consequence newlines are collapsed as one unless multiple <br> tags are used.
-func html2Text(htmlDocument string) (string, error) {
- doc, err := html.Parse(strings.NewReader(htmlDocument))
- if err != nil {
- return "", err
- }
-
- var builder strings.Builder
- var canAddNewLine bool
-
- // see https://pkg.go.dev/golang.org/x/net/html#Parse
- var f func(*html.Node, *strings.Builder)
- f = func(n *html.Node, activeBuilder *strings.Builder) {
- isLink := n.Type == html.ElementNode && n.Data == "a"
-
- if isLink {
- var linkBuilder strings.Builder
- activeBuilder = &linkBuilder
- } else if activeBuilder == nil {
- activeBuilder = &builder
- }
-
- switch n.Type {
- case html.TextNode:
- txt := whitespaceRegex.ReplaceAllString(n.Data, " ")
-
- // the prev node has new line so it is safe to trim the indentation
- if !canAddNewLine {
- txt = strings.TrimLeft(txt, " ")
- }
-
- if txt != "" {
- activeBuilder.WriteString(txt)
- canAddNewLine = true
- }
- case html.ElementNode:
- if n.Data == "br" {
- // always write new lines when <br> tag is used
- activeBuilder.WriteString("\r\n")
- canAddNewLine = false
- }
- // prefix list items with dash
- if n.Data == "li" {
- activeBuilder.WriteString("- ")
- }
- }
-
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- if c.Type != html.ElementNode {
- f(c, activeBuilder)
- }
- }
-
- // format links as [label](href)
- if isLink {
- linkTxt := strings.TrimSpace(activeBuilder.String())
- if linkTxt == "" {
- linkTxt = "LINK"
- }
-
- builder.WriteString("[")
- builder.WriteString(linkTxt)
- builder.WriteString("]")
-
- // link href attr extraction
- for _, a := range n.Attr {
- if a.Key == "href" {
- if a.Val != "" {
- builder.WriteString("(")
- builder.WriteString(a.Val)
- builder.WriteString(")")
- }
- break
- }
- }
-
- activeBuilder.Reset()
- }
- }
-
- f(doc, &builder)
-
- return strings.TrimSpace(builder.String()), nil
-}
diff --git a/internal/mailer/mailer.go b/internal/mailer/mailer.go
deleted file mode 100644
index b64da89..0000000
--- a/internal/mailer/mailer.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package mailer
-
-import (
- "io"
- "net/mail"
-)
-
-// Message defines a generic email message struct.
-type Message struct {
- From mail.Address `json:"from"`
- To []mail.Address `json:"to"`
- Bcc []mail.Address `json:"bcc"`
- Cc []mail.Address `json:"cc"`
- Subject string `json:"subject"`
- HTML string `json:"html"`
- Text string `json:"text"`
- Headers map[string]string `json:"headers"`
- Attachments map[string]io.Reader `json:"attachments"`
-}
-
-// Mailer defines a base mail client interface.
-type Mailer interface {
- // Send sends an email with the provided Message.
- Send(message *Message) error
-}
-
-// addressesToStrings converts the provided address to a list of serialized RFC 5322 strings.
-//
-// To export only the email part of mail.Address, you can set withName to false.
-func addressesToStrings(addresses []mail.Address, withName bool) []string {
- result := make([]string, len(addresses))
-
- for i, addr := range addresses {
- if withName && addr.Name != "" {
- result[i] = addr.String()
- } else {
- // keep only the email part to avoid wrapping in angle-brackets
- result[i] = addr.Address
- }
- }
-
- return result
-}
diff --git a/internal/mailer/sendmail.go b/internal/mailer/sendmail.go
deleted file mode 100644
index 6826a03..0000000
--- a/internal/mailer/sendmail.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package mailer
-
-import (
- "bytes"
- "errors"
- "mime"
- "net/http"
- "os/exec"
- "strings"
-)
-
-var _ Mailer = (*Sendmail)(nil)
-
-// Sendmail implements [mailer.Mailer] interface and defines a mail
-// client that sends emails via the "sendmail" *nix command.
-//
-// This client is usually recommended only for development and testing.
-type Sendmail struct {
-}
-
-// Send implements `mailer.Mailer` interface.
-func (c *Sendmail) Send(m *Message) error {
- toAddresses := addressesToStrings(m.To, false)
-
- headers := make(http.Header)
- headers.Set("Subject", mime.QEncoding.Encode("utf-8", m.Subject))
- headers.Set("From", m.From.String())
- headers.Set("Content-Type", "text/html; charset=UTF-8")
- headers.Set("To", strings.Join(toAddresses, ","))
-
- cmdPath, err := findSendmailPath()
- if err != nil {
- return err
- }
-
- var buffer bytes.Buffer
-
- // write
- // ---
- if err := headers.Write(&buffer); err != nil {
- return err
- }
- if _, err := buffer.Write([]byte("\r\n")); err != nil {
- return err
- }
- if m.HTML != "" {
- if _, err := buffer.Write([]byte(m.HTML)); err != nil {
- return err
- }
- } else {
- if _, err := buffer.Write([]byte(m.Text)); err != nil {
- return err
- }
- }
- // ---
-
- sendmail := exec.Command(cmdPath, strings.Join(toAddresses, ","))
- sendmail.Stdin = &buffer
-
- return sendmail.Run()
-}
-
-func findSendmailPath() (string, error) {
- options := []string{
- "/usr/sbin/sendmail",
- "/usr/bin/sendmail",
- "sendmail",
- }
-
- for _, option := range options {
- path, err := exec.LookPath(option)
- if err == nil {
- return path, err
- }
- }
-
- return "", errors.New("failed to locate a sendmail executable path")
-}
diff --git a/internal/mailer/smtp.go b/internal/mailer/smtp.go
deleted file mode 100644
index 58bbade..0000000
--- a/internal/mailer/smtp.go
+++ /dev/null
@@ -1,196 +0,0 @@
-package mailer
-
-import (
- "errors"
- "fmt"
- "net/smtp"
- "strings"
-
- "github.com/domodwyer/mailyak/v3"
- "github.com/google/uuid"
-)
-
-var _ Mailer = (*SmtpClient)(nil)
-
-const (
- SmtpAuthPlain = "PLAIN"
- SmtpAuthLogin = "LOGIN"
-)
-
-// Deprecated: Use directly the SmtpClient struct literal.
-//
-// NewSmtpClient creates new SmtpClient with the provided configuration.
-func NewSmtpClient(
- host string,
- port int,
- username string,
- password string,
- tls bool,
-) *SmtpClient {
- return &SmtpClient{
- Host: host,
- Port: port,
- Username: username,
- Password: password,
- Tls: tls,
- }
-}
-
-// SmtpClient defines a SMTP mail client structure that implements
-// `mailer.Mailer` interface.
-type SmtpClient struct {
- Host string
- Port int
- Username string
- Password string
- Tls bool
-
- // SMTP auth method to use
- // (if not explicitly set, defaults to "PLAIN")
- AuthMethod string
-
- // LocalName is optional domain name used for the EHLO/HELO exchange
- // (if not explicitly set, defaults to "localhost").
- //
- // This is required only by some SMTP servers, such as Gmail SMTP-relay.
- LocalName string
-}
-
-// Send implements `mailer.Mailer` interface.
-func (c *SmtpClient) Send(m *Message) error {
- var smtpAuth smtp.Auth
- if c.Username != "" || c.Password != "" {
- switch c.AuthMethod {
- case SmtpAuthLogin:
- smtpAuth = &smtpLoginAuth{c.Username, c.Password}
- default:
- smtpAuth = smtp.PlainAuth("", c.Username, c.Password, c.Host)
- }
- }
-
- // create mail instance
- var yak *mailyak.MailYak
- if c.Tls {
- var tlsErr error
- yak, tlsErr = mailyak.NewWithTLS(fmt.Sprintf("%s:%d", c.Host, c.Port), smtpAuth, nil)
- if tlsErr != nil {
- return tlsErr
- }
- } else {
- yak = mailyak.New(fmt.Sprintf("%s:%d", c.Host, c.Port), smtpAuth)
- }
-
- if c.LocalName != "" {
- yak.LocalName(c.LocalName)
- }
-
- if m.From.Name != "" {
- yak.FromName(m.From.Name)
- }
- yak.From(m.From.Address)
- yak.Subject(m.Subject)
- yak.HTML().Set(m.HTML)
-
- if m.Text == "" {
- // try to generate a plain text version of the HTML
- if plain, err := html2Text(m.HTML); err == nil {
- yak.Plain().Set(plain)
- }
- } else {
- yak.Plain().Set(m.Text)
- }
-
- if len(m.To) > 0 {
- yak.To(addressesToStrings(m.To, true)...)
- }
-
- if len(m.Bcc) > 0 {
- yak.Bcc(addressesToStrings(m.Bcc, true)...)
- }
-
- if len(m.Cc) > 0 {
- yak.Cc(addressesToStrings(m.Cc, true)...)
- }
-
- // add attachements (if any)
- for name, data := range m.Attachments {
- yak.Attach(name, data)
- }
-
- // add custom headers (if any)
- var hasMessageId bool
- for k, v := range m.Headers {
- if strings.EqualFold(k, "Message-ID") {
- hasMessageId = true
- }
- yak.AddHeader(k, v)
- }
- if !hasMessageId {
- // add a default message id if missing
- fromParts := strings.Split(m.From.Address, "@")
- if len(fromParts) == 2 {
- yak.AddHeader("Message-ID", fmt.Sprintf("<%s@%s>",
- uuid.New().String(),
- fromParts[1],
- ))
- }
- }
-
- return yak.Send()
-}
-
-// -------------------------------------------------------------------
-// AUTH LOGIN
-// -------------------------------------------------------------------
-
-var _ smtp.Auth = (*smtpLoginAuth)(nil)
-
-// smtpLoginAuth defines an AUTH that implements the LOGIN authentication mechanism.
-//
-// AUTH LOGIN is obsolete[1] but some mail services like outlook requires it [2].
-//
-// NB!
-// It will only send the credentials if the connection is using TLS or is connected to localhost.
-// Otherwise authentication will fail with an error, without sending the credentials.
-//
-// [1]: https://github.com/golang/go/issues/40817
-// [2]: https://support.microsoft.com/en-us/office/outlook-com-no-longer-supports-auth-plain-authentication-07f7d5e9-1697-465f-84d2-4513d4ff0145?ui=en-us&rs=en-us&ad=us
-type smtpLoginAuth struct {
- username, password string
-}
-
-// Start initializes an authentication with the server.
-//
-// It is part of the [smtp.Auth] interface.
-func (a *smtpLoginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
- // Must have TLS, or else localhost server.
- // Note: If TLS is not true, then we can't trust ANYTHING in ServerInfo.
- // In particular, it doesn't matter if the server advertises LOGIN auth.
- // That might just be the attacker saying
- // "it's ok, you can trust me with your password."
- if !server.TLS && !isLocalhost(server.Name) {
- return "", nil, errors.New("unencrypted connection")
- }
-
- return "LOGIN", nil, nil
-}
-
-// Next "continues" the auth process by feeding the server with the requested data.
-//
-// It is part of the [smtp.Auth] interface.
-func (a *smtpLoginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
- if more {
- switch strings.ToLower(string(fromServer)) {
- case "username:":
- return []byte(a.username), nil
- case "password:":
- return []byte(a.password), nil
- }
- }
-
- return nil, nil
-}
-
-func isLocalhost(name string) bool {
- return name == "localhost" || name == "127.0.0.1" || name == "::1"
-}
diff --git a/internal/mailer/smtp_test.go b/internal/mailer/smtp_test.go
deleted file mode 100644
index 072ae14..0000000
--- a/internal/mailer/smtp_test.go
+++ /dev/null
@@ -1,164 +0,0 @@
-package mailer
-
-import (
- "net/smtp"
- "testing"
-)
-
-func TestLoginAuthStart(t *testing.T) {
- auth := smtpLoginAuth{username: "test", password: "123456"}
-
- scenarios := []struct {
- name string
- serverInfo *smtp.ServerInfo
- expectError bool
- }{
- {
- "localhost without tls",
- &smtp.ServerInfo{TLS: false, Name: "localhost"},
- false,
- },
- {
- "localhost with tls",
- &smtp.ServerInfo{TLS: true, Name: "localhost"},
- false,
- },
- {
- "127.0.0.1 without tls",
- &smtp.ServerInfo{TLS: false, Name: "127.0.0.1"},
- false,
- },
- {
- "127.0.0.1 with tls",
- &smtp.ServerInfo{TLS: false, Name: "127.0.0.1"},
- false,
- },
- {
- "::1 without tls",
- &smtp.ServerInfo{TLS: false, Name: "::1"},
- false,
- },
- {
- "::1 with tls",
- &smtp.ServerInfo{TLS: false, Name: "::1"},
- false,
- },
- {
- "non-localhost without tls",
- &smtp.ServerInfo{TLS: false, Name: "example.com"},
- true,
- },
- {
- "non-localhost with tls",
- &smtp.ServerInfo{TLS: true, Name: "example.com"},
- false,
- },
- }
-
- for _, s := range scenarios {
- method, resp, err := auth.Start(s.serverInfo)
-
- hasErr := err != nil
- if hasErr != s.expectError {
- t.Fatalf("[%s] Expected hasErr %v, got %v", s.name, s.expectError, hasErr)
- }
-
- if hasErr {
- continue
- }
-
- if len(resp) != 0 {
- t.Fatalf("[%s] Expected empty data response, got %v", s.name, resp)
- }
-
- if method != "LOGIN" {
- t.Fatalf("[%s] Expected LOGIN, got %v", s.name, method)
- }
- }
-}
-
-func TestLoginAuthNext(t *testing.T) {
- auth := smtpLoginAuth{username: "test", password: "123456"}
-
- {
- // example|false
- r1, err := auth.Next([]byte("example:"), false)
- if err != nil {
- t.Fatalf("[example|false] Unexpected error %v", err)
- }
- if len(r1) != 0 {
- t.Fatalf("[example|false] Expected empty part, got %v", r1)
- }
-
- // example|true
- r2, err := auth.Next([]byte("example:"), true)
- if err != nil {
- t.Fatalf("[example|true] Unexpected error %v", err)
- }
- if len(r2) != 0 {
- t.Fatalf("[example|true] Expected empty part, got %v", r2)
- }
- }
-
- // ---------------------------------------------------------------
-
- {
- // username:|false
- r1, err := auth.Next([]byte("username:"), false)
- if err != nil {
- t.Fatalf("[username|false] Unexpected error %v", err)
- }
- if len(r1) != 0 {
- t.Fatalf("[username|false] Expected empty part, got %v", r1)
- }
-
- // username:|true
- r2, err := auth.Next([]byte("username:"), true)
- if err != nil {
- t.Fatalf("[username|true] Unexpected error %v", err)
- }
- if str := string(r2); str != auth.username {
- t.Fatalf("[username|true] Expected %s, got %s", auth.username, str)
- }
-
- // uSeRnAmE:|true
- r3, err := auth.Next([]byte("uSeRnAmE:"), true)
- if err != nil {
- t.Fatalf("[uSeRnAmE|true] Unexpected error %v", err)
- }
- if str := string(r3); str != auth.username {
- t.Fatalf("[uSeRnAmE|true] Expected %s, got %s", auth.username, str)
- }
- }
-
- // ---------------------------------------------------------------
-
- {
- // password:|false
- r1, err := auth.Next([]byte("password:"), false)
- if err != nil {
- t.Fatalf("[password|false] Unexpected error %v", err)
- }
- if len(r1) != 0 {
- t.Fatalf("[password|false] Expected empty part, got %v", r1)
- }
-
- // password:|true
- r2, err := auth.Next([]byte("password:"), true)
- if err != nil {
- t.Fatalf("[password|true] Unexpected error %v", err)
- }
- if str := string(r2); str != auth.password {
- t.Fatalf("[password|true] Expected %s, got %s", auth.password, str)
- }
-
- // pAsSwOrD:|true
- r3, err := auth.Next([]byte("pAsSwOrD:"), true)
- if err != nil {
- t.Fatalf("[pAsSwOrD|true] Unexpected error %v", err)
- }
- if str := string(r3); str != auth.password {
- t.Fatalf("[pAsSwOrD|true] Expected %s, got %s", auth.password, str)
- }
- }
-}
diff --git a/plugin/httpgetter/html_meta.go b/plugin/httpgetter/html_meta.go
new file mode 100644
index 0000000..f69bad6
--- /dev/null
+++ b/plugin/httpgetter/html_meta.go
@@ -0,0 +1,98 @@
+package httpgetter
+
+import (
+ "errors"
+ "io"
+ "net/http"
+ "net/url"
+
+ "golang.org/x/net/html"
+ "golang.org/x/net/html/atom"
+)
+
+type HTMLMeta struct {
+ Title string `json:"title"`
+ Description string `json:"description"`
+ Image string `json:"image"`
+}
+
+func GetHTMLMeta(urlStr string) (*HTMLMeta, error) {
+ if _, err := url.Parse(urlStr); err != nil {
+ return nil, err
+ }
+
+ response, err := http.Get(urlStr)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ mediatype, err := getMediatype(response)
+ if err != nil {
+ return nil, err
+ }
+ if mediatype != "text/html" {
+ return nil, errors.New("not a HTML page")
+ }
+
+ htmlMeta := extractHTMLMeta(response.Body)
+ return htmlMeta, nil
+}
+
+func extractHTMLMeta(resp io.Reader) *HTMLMeta {
+ tokenizer := html.NewTokenizer(resp)
+ htmlMeta := new(HTMLMeta)
+
+ for {
+ tokenType := tokenizer.Next()
+ if tokenType == html.ErrorToken {
+ break
+ } else if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken {
+ token := tokenizer.Token()
+ if token.DataAtom == atom.Body {
+ break
+ }
+
+ if token.DataAtom == atom.Title {
+ tokenizer.Next()
+ token := tokenizer.Token()
+ htmlMeta.Title = token.Data
+ } else if token.DataAtom == atom.Meta {
+ description, ok := extractMetaProperty(token, "description")
+ if ok {
+ htmlMeta.Description = description
+ }
+
+ ogTitle, ok := extractMetaProperty(token, "og:title")
+ if ok {
+ htmlMeta.Title = ogTitle
+ }
+
+ ogDescription, ok := extractMetaProperty(token, "og:description")
+ if ok {
+ htmlMeta.Description = ogDescription
+ }
+
+ ogImage, ok := extractMetaProperty(token, "og:image")
+ if ok {
+ htmlMeta.Image = ogImage
+ }
+ }
+ }
+ }
+
+ return htmlMeta
+}
+
+func extractMetaProperty(token html.Token, prop string) (content string, ok bool) {
+ content, ok = "", false
+ for _, attr := range token.Attr {
+ if attr.Key == "property" && attr.Val == prop {
+ ok = true
+ }
+ if attr.Key == "content" {
+ content = attr.Val
+ }
+ }
+ return content, ok
+}
diff --git a/plugin/httpgetter/html_meta_test.go b/plugin/httpgetter/html_meta_test.go
new file mode 100644
index 0000000..45345d3
--- /dev/null
+++ b/plugin/httpgetter/html_meta_test.go
@@ -0,0 +1,19 @@
+package httpgetter
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetHTMLMeta(t *testing.T) {
+ tests := []struct {
+ urlStr string
+ htmlMeta HTMLMeta
+ }{}
+ for _, test := range tests {
+ metadata, err := GetHTMLMeta(test.urlStr)
+ require.NoError(t, err)
+ require.Equal(t, test.htmlMeta, *metadata)
+ }
+}
diff --git a/plugin/httpgetter/http_getter.go b/plugin/httpgetter/http_getter.go
new file mode 100644
index 0000000..c545baf
--- /dev/null
+++ b/plugin/httpgetter/http_getter.go
@@ -0,0 +1,4 @@
+// Package httpgetter is using to get resources from url.
+// * Get metadata for website;
+// * Get image blob to avoid CORS;
+package httpgetter
diff --git a/plugin/httpgetter/image.go b/plugin/httpgetter/image.go
new file mode 100644
index 0000000..2d6a163
--- /dev/null
+++ b/plugin/httpgetter/image.go
@@ -0,0 +1,45 @@
+package httpgetter
+
+import (
+ "errors"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+)
+
+type Image struct {
+ Blob []byte
+ Mediatype string
+}
+
+func GetImage(urlStr string) (*Image, error) {
+ if _, err := url.Parse(urlStr); err != nil {
+ return nil, err
+ }
+
+ response, err := http.Get(urlStr)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ mediatype, err := getMediatype(response)
+ if err != nil {
+ return nil, err
+ }
+ if !strings.HasPrefix(mediatype, "image/") {
+ return nil, errors.New("Wrong image mediatype")
+ }
+
+ bodyBytes, err := io.ReadAll(response.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ image := &Image{
+ Blob: bodyBytes,
+ Mediatype: mediatype,
+ }
+ return image, nil
+}
diff --git a/plugin/httpgetter/util.go b/plugin/httpgetter/util.go
new file mode 100644
index 0000000..d83f5ef
--- /dev/null
+++ b/plugin/httpgetter/util.go
@@ -0,0 +1,15 @@
+package httpgetter
+
+import (
+ "mime"
+ "net/http"
+)
+
+func getMediatype(response *http.Response) (string, error) {
+ contentType := response.Header.Get("content-type")
+ mediatype, _, err := mime.ParseMediaType(contentType)
+ if err != nil {
+ return "", err
+ }
+ return mediatype, nil
+}