feat: courier template configs (#2156)

It is now possible to override individual courier email templates using the configuration system!

Closes #2054

README.md

@@ -165,7 +165,7 @@ that your company deserves a spot here, reach out to
             <td>DataDetect</td>
             <td align="center"><img height="32px" src="https://raw.githubusercontent.com/ory/meta/master/static/adopters/datadetect.svg" alt="Datadetect"></td>
             <td><a href="https://unifiedglobalarchiving.com/data-detect/">unifiedglobalarchiving.com/data-detect/</a></td>
-        </tr>        
+        </tr>
         <tr>
             <td>Adopter *</td>
             <td>Sainsbury's</td>
@@ -183,13 +183,13 @@ that your company deserves a spot here, reach out to
             <td>Reyah</td>
             <td align="center"><img height="32px" src="https://raw.githubusercontent.com/ory/meta/master/static/adopters/reyah.svg" alt="Reyah"></td>
             <td><a href="https://reyah.eu/">reyah.eu</a></td>
-        </tr>        
+        </tr>
         <tr>
             <td>Adopter *</td>
             <td>Zero</td>
             <td align="center"><img height="32px" src="https://raw.githubusercontent.com/ory/meta/master/static/adopters/commitzero.svg" alt="Project Zero by Commit"></td>
             <td><a href="https://getzero.dev/">getzero.dev</a></td>
-        </tr>        
+        </tr>
         <tr>
             <td>Adopter *</td>
             <td>Padis</td>
@@ -207,7 +207,7 @@ that your company deserves a spot here, reach out to
             <td>Security Onion Solutions</td>
             <td align="center"><img height="32px" src="https://raw.githubusercontent.com/ory/meta/master/static/adopters/securityonion.svg" alt="Security Onion Solutions"></td>
             <td><a href="https://securityonionsolutions.com/">securityonionsolutions.com</a></td>
-        </tr>        
+        </tr>
         <tr>
             <td>Adopter *</td>
             <td>Factly</td>
@@ -231,7 +231,7 @@ that your company deserves a spot here, reach out to
             <td>Spiri.bo</td>
             <td align="center"><img height="32px" src="https://raw.githubusercontent.com/ory/meta/master/static/adopters/spiribo.svg" alt="Spiri.bo"></td>
             <td><a href="https://spiri.bo/">spiri.bo</a></td>
-        </tr>        
+        </tr>
         <tr>
             <td>Sponsor</td>
             <td>Strivacity</td>
@@ -504,6 +504,30 @@ For more details, run:
 ./test/e2e/run.sh
 </pre>
 
+**Run only a singular test**
+
+Add `.only` to the test you would like to run.
+
+For example:
+
+```ts
+it.only('invalid remote recovery email template', () => {
+    ...
+})
+```
+
+**Run a subset of tests**
+
+This will require editing the `cypress.json` file located in the `test/e2e/` folder.
+
+Add the `testFiles` option and specify the test to run inside the `cypress/integration` folder.
+As an example we will add only the `network` tests.
+```json
+"testFiles": ["profiles/network/*"],
+```
+
+Now start the tests again using the run script or makefile.
+
 #### Build Docker
 
 You can build a development Docker Image using:

courier/courier.go

@@ -5,10 +5,14 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"net/url"
 	"strconv"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
+
+	"github.com/ory/kratos/driver/config"
+	"github.com/ory/x/httpx"
+
 	"github.com/cenkalti/backoff"
 	"github.com/gofrs/uuid"
 	"github.com/pkg/errors"
@@ -21,20 +25,14 @@ import (
 )
 
 type (
-	SMTPConfig interface {
-		CourierSMTPURL() *url.URL
-		CourierSMTPFrom() string
-		CourierSMTPFromName() string
-		CourierSMTPHeaders() map[string]string
-		CourierTemplatesRoot() string
-	}
 	SMTPDependencies interface {
 		PersistenceProvider
 		x.LoggingProvider
 		ConfigProvider
+		HTTPClient(ctx context.Context, opts ...httpx.ResilientOptions) *retryablehttp.Client
 	}
 	TemplateTyper            func(t EmailTemplate) (TemplateType, error)
-	EmailTemplateFromMessage func(c SMTPConfig, msg Message) (EmailTemplate, error)
+	EmailTemplateFromMessage func(d SMTPDependencies, msg Message) (EmailTemplate, error)
 	Courier                  struct {
 		Dialer                      *gomail.Dialer
 		d                           SMTPDependencies
@@ -45,7 +43,7 @@ type (
 		Courier(ctx context.Context) *Courier
 	}
 	ConfigProvider interface {
-		CourierConfig(ctx context.Context) SMTPConfig
+		CourierConfig(ctx context.Context) config.CourierConfigs
 	}
 )
 
@@ -101,12 +99,12 @@ func (m *Courier) QueueEmail(ctx context.Context, t EmailTemplate) (uuid.UUID, e
 		return uuid.Nil, err
 	}
 
-	subject, err := t.EmailSubject()
+	subject, err := t.EmailSubject(ctx)
 	if err != nil {
 		return uuid.Nil, err
 	}
 
-	bodyPlaintext, err := t.EmailBodyPlaintext()
+	bodyPlaintext, err := t.EmailBodyPlaintext(ctx)
 	if err != nil {
 		return uuid.Nil, err
 	}
@@ -188,14 +186,14 @@ func (m *Courier) DispatchMessage(ctx context.Context, msg Message) error {
 
 		gm.SetBody("text/plain", msg.Body)
 
-		tmpl, err := m.NewEmailTemplateFromMessage(m.d.CourierConfig(ctx), msg)
+		tmpl, err := m.NewEmailTemplateFromMessage(m.d, msg)
 		if err != nil {
 			m.d.Logger().
 				WithError(err).
 				WithField("message_id", msg.ID).
 				Error(`Unable to get email template from message.`)
 		} else {
-			htmlBody, err := tmpl.EmailBody()
+			htmlBody, err := tmpl.EmailBody(ctx)
 			if err != nil {
 				m.d.Logger().
 					WithError(err).

courier/courier_test.go

@@ -85,15 +85,15 @@ func TestSMTP(t *testing.T) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	id, err := c.QueueEmail(ctx, templates.NewTestStub(conf, &templates.TestStubModel{
+	id, err := c.QueueEmail(ctx, templates.NewTestStub(reg, &templates.TestStubModel{
 		To:      "[email protected]",
 		Subject: "test-subject-1",
 		Body:    "test-body-1",
 	}))
 	require.NoError(t, err)
 	require.NotEqual(t, uuid.Nil, id)
 
-	id, err = c.QueueEmail(ctx, templates.NewTestStub(conf, &templates.TestStubModel{
+	id, err = c.QueueEmail(ctx, templates.NewTestStub(reg, &templates.TestStubModel{
 		To:      "[email protected]",
 		Subject: "test-subject-2",
 		Body:    "test-body-2",
@@ -107,7 +107,7 @@ func TestSMTP(t *testing.T) {
 	conf.MustSet(config.ViperKeyCourierSMTPHeaders+".test-stub-header2", "bar")
 	customerHeaders := conf.CourierSMTPHeaders()
 	require.Len(t, customerHeaders, 2)
-	id, err = c.QueueEmail(ctx, templates.NewTestStub(conf, &templates.TestStubModel{
+	id, err = c.QueueEmail(ctx, templates.NewTestStub(reg, &templates.TestStubModel{
 		To:      "[email protected]",
 		Subject: "test-subject-3",
 		Body:    "test-body-3",

courier/template/load_template.go

@@ -2,13 +2,19 @@ package template
 
 import (
 	"bytes"
+	"context"
 	"embed"
 	htemplate "html/template"
 	"io"
 	"io/fs"
 	"path/filepath"
 	"text/template"
 
+	"github.com/hashicorp/go-retryablehttp"
+
+	"github.com/ory/x/fetcher"
+	"github.com/ory/x/httpx"
+
 	"github.com/Masterminds/sprig/v3"
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/pkg/errors"
@@ -17,18 +23,22 @@ import (
 //go:embed courier/builtin/templates/*
 var templates embed.FS
 
-var cache, _ = lru.New(16)
+var Cache, _ = lru.New(16)
 
 type Template interface {
 	Execute(wr io.Writer, data interface{}) error
 }
 
-func loadBuiltInTemplate(filesytem fs.FS, name string, html bool) (Template, error) {
-	if t, found := cache.Get(name); found {
+type templateDependencies interface {
+	HTTPClient(ctx context.Context, opts ...httpx.ResilientOptions) *retryablehttp.Client
+}
+
+func loadBuiltInTemplate(filesystem fs.FS, name string, html bool) (Template, error) {
+	if t, found := Cache.Get(name); found {
 		return t.(Template), nil
 	}
 
-	file, err := filesytem.Open(name)
+	file, err := filesystem.Open(name)
 	if err != nil {
 		// try to fallback to bundled templates
 		var fallbackErr error
@@ -61,12 +71,45 @@ func loadBuiltInTemplate(filesytem fs.FS, name string, html bool) (Template, err
 		tpl = t
 	}
 
-	_ = cache.Add(name, tpl)
+	_ = Cache.Add(name, tpl)
 	return tpl, nil
 }
 
+func loadRemoteTemplate(ctx context.Context, d templateDependencies, url string, html bool) (Template, error) {
+	var b []byte
+	var err error
+
+	// instead of creating a new request always we always cache the bytes.Buffer using the url as the key
+	if t, found := Cache.Get(url); found {
+		b = t.([]byte)
+	} else {
+		f := fetcher.NewFetcher(fetcher.WithClient(d.HTTPClient(ctx)))
+		bb, err := f.Fetch(url)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		b = bb.Bytes()
+		_ = Cache.Add(url, b)
+	}
+
+	var t Template
+	if html {
+		t, err = htemplate.New(url).Funcs(sprig.HtmlFuncMap()).Parse(string(b))
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+	} else {
+		t, err = template.New(url).Funcs(sprig.TxtFuncMap()).Parse(string(b))
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+	}
+
+	return t, nil
+}
+
 func loadTemplate(filesystem fs.FS, name, pattern string, html bool) (Template, error) {
-	if t, found := cache.Get(name); found {
+	if t, found := Cache.Get(name); found {
 		return t.(Template), nil
 	}
 
@@ -102,15 +145,23 @@ func loadTemplate(filesystem fs.FS, name, pattern string, html bool) (Template,
 		tpl = t
 	}
 
-	_ = cache.Add(name, tpl)
+	_ = Cache.Add(name, tpl)
 	return tpl, nil
 }
 
-func LoadTextTemplate(filesystem fs.FS, name, pattern string, model interface{}) (string, error) {
-	t, err := loadTemplate(filesystem, name, pattern, false)
-
-	if err != nil {
-		return "", err
+func LoadTextTemplate(ctx context.Context, d templateDependencies, filesystem fs.FS, name, pattern string, model interface{}, remoteURL string) (string, error) {
+	var t Template
+	var err error
+	if remoteURL != "" {
+		t, err = loadRemoteTemplate(ctx, d, remoteURL, false)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		t, err = loadTemplate(filesystem, name, pattern, false)
+		if err != nil {
+			return "", err
+		}
 	}
 
 	var b bytes.Buffer
@@ -120,11 +171,19 @@ func LoadTextTemplate(filesystem fs.FS, name, pattern string, model interface{})
 	return b.String(), nil
 }
 
-func LoadHTMLTemplate(filesystem fs.FS, name, pattern string, model interface{}) (string, error) {
-	t, err := loadTemplate(filesystem, name, pattern, true)
-
-	if err != nil {
-		return "", err
+func LoadHTMLTemplate(ctx context.Context, d templateDependencies, filesystem fs.FS, name, pattern string, model interface{}, remoteURL string) (string, error) {
+	var t Template
+	var err error
+	if remoteURL != "" {
+		t, err = loadRemoteTemplate(ctx, d, remoteURL, true)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		t, err = loadTemplate(filesystem, name, pattern, true)
+		if err != nil {
+			return "", err
+		}
 	}
 
 	var b bytes.Buffer