Mime cache (#59)

* support different caching duration for different mime types #58

* extract main code to run func

* lint: err shadowing
This commit is contained in:
Umputun
2021-04-26 18:51:48 -05:00
committed by GitHub
parent f5c5c61349
commit a4cffbe922
7 changed files with 377 additions and 120 deletions

View File

@@ -54,7 +54,7 @@ linters:
- varcheck
- stylecheck
- gochecknoinits
- scopelint
- exportloopref
- gocritic
- nakedret
- gosimple

View File

@@ -126,6 +126,10 @@ In addition to the common assets server, multiple custom static servers are supp
Assets server supports caching control with the `--assets.cache=<duration>` parameter. `0s` duration (default) turns caching control off. A duration is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
There are two ways to set cache duration:
1. A single value for all static assets. This is as simple as `--assets.cache=48h`.
2. Custom duration for different mime types. It should include two parts - the default value and the pairs of mime:duration. In command line this looks like multiple `--assets.cache` options, i.e. `--assets.cache=48h --assets.cache=text/html:24h --assets.cache=image/png:2h`. Environment values should be comma-separated, i.e. `ASSETS_CACHE=48h,text/html:24h,image/png:2h`
## More options
- `--gzip` enables gzip compression for responses.

View File

@@ -40,9 +40,9 @@ var opts struct {
} `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
Assets struct {
Location string `short:"a" long:"location" env:"LOCATION" default:"" description:"assets location"`
WebRoot string `long:"root" env:"ROOT" default:"/" description:"assets web root"`
CacheDuration time.Duration `long:"cache" env:"CACHE" default:"0s" description:"cache duration for assets"`
Location string `short:"a" long:"location" env:"LOCATION" default:"" description:"assets location"`
WebRoot string `long:"root" env:"ROOT" default:"/" description:"assets web root"`
CacheControl []string `long:"cache" env:"CACHE" description:"cache duration for assets" env-delim:","`
} `group:"assets" namespace:"assets" env-namespace:"ASSETS"`
Logger struct {
@@ -111,8 +111,23 @@ func main() {
setupLog(opts.Dbg)
log.Printf("[DEBUG] options: %+v", opts)
err := run()
if err != nil {
log.Fatalf("[ERROR] proxy server failed, %v", err)
}
}
func run() error {
ctx, cancel := context.WithCancel(context.Background())
go func() { // catch signal and invoke graceful termination
go func() {
if x := recover(); x != nil {
log.Printf("[WARN] run time panic:\n%v", x)
panic(x)
}
// catch signal and invoke graceful termination
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
<-stop
@@ -122,38 +137,31 @@ func main() {
providers, err := makeProviders()
if err != nil {
log.Fatalf("[ERROR] failed to make providers, %v", err)
return fmt.Errorf("failed to make providers: %w", err)
}
svc := discovery.NewService(providers, time.Second)
if len(providers) > 0 {
go func() {
if e := svc.Run(context.Background()); e != nil {
log.Fatalf("[ERROR] discovery failed, %v", e)
log.Printf("[WARN] discovery failed, %v", e)
}
}()
}
sslConfig, err := makeSSLConfig()
if err != nil {
log.Fatalf("[ERROR] failed to make config of ssl server params, %v", err)
sslConfig, sslErr := makeSSLConfig()
if sslErr != nil {
return fmt.Errorf("failed to make config of ssl server params: %w", sslErr)
}
defer func() {
if x := recover(); x != nil {
log.Printf("[WARN] run time panic:\n%v", x)
panic(x)
}
}()
accessLog := makeAccessLogWriter()
defer func() {
if err := accessLog.Close(); err != nil {
log.Printf("[WARN] can't close access log, %v", err)
if logErr := accessLog.Close(); logErr != nil {
log.Printf("[WARN] can't close access log, %v", logErr)
}
}()
metrcis := mgmt.NewMetrics()
metrics := mgmt.NewMetrics()
go func() {
mgSrv := mgmt.Server{
Listen: opts.Management.Listen,
@@ -161,29 +169,34 @@ func main() {
AssetsLocation: opts.Assets.Location,
AssetsWebRoot: opts.Assets.WebRoot,
Version: revision,
Metrics: metrcis,
Metrics: metrics,
}
if opts.Management.Enabled {
if err := mgSrv.Run(ctx); err != nil {
log.Printf("[WARN] management service failed, %v", err)
if mgErr := mgSrv.Run(ctx); err != nil {
log.Printf("[WARN] management service failed, %v", mgErr)
}
}
}()
cacheControl, err := proxy.MakeCacheControl(opts.Assets.CacheControl)
if err != nil {
return fmt.Errorf("failed to make cache control: %w", err)
}
px := &proxy.Http{
Version: revision,
Matcher: svc,
Address: opts.Listen,
MaxBodySize: opts.MaxSize,
AssetsLocation: opts.Assets.Location,
AssetsWebRoot: opts.Assets.WebRoot,
AssetsCacheDuration: opts.Assets.CacheDuration,
GzEnabled: opts.GzipEnabled,
SSLConfig: sslConfig,
ProxyHeaders: opts.ProxyHeaders,
AccessLog: accessLog,
StdOutEnabled: opts.Logger.StdOut,
Signature: opts.Signature,
Version: revision,
Matcher: svc,
Address: opts.Listen,
MaxBodySize: opts.MaxSize,
AssetsLocation: opts.Assets.Location,
AssetsWebRoot: opts.Assets.WebRoot,
CacheControl: cacheControl,
GzEnabled: opts.GzipEnabled,
SSLConfig: sslConfig,
ProxyHeaders: opts.ProxyHeaders,
AccessLog: accessLog,
StdOutEnabled: opts.Logger.StdOut,
Signature: opts.Signature,
Timeouts: proxy.Timeouts{
ReadHeader: opts.Timeouts.ReadHeader,
Write: opts.Timeouts.Write,
@@ -195,15 +208,15 @@ func main() {
ExpectContinue: opts.Timeouts.ExpectContinue,
ResponseHeader: opts.Timeouts.ResponseHeader,
},
Metrics: metrcis,
Metrics: metrics,
}
if err := px.Run(ctx); err != nil {
if err == http.ErrServerClosed {
log.Printf("[WARN] proxy server closed, %v", err) //nolint gocritic
return
}
log.Fatalf("[ERROR] proxy server failed, %v", err) //nolint gocritic
err = px.Run(ctx)
if err != nil && err == http.ErrServerClosed {
log.Printf("[WARN] proxy server closed, %v", err) //nolint gocritic
return nil
}
return err
}
// make all providers. the order is matter, defines which provider will have priority in case of conflicting rules

113
app/proxy/cache_control.go Normal file
View File

@@ -0,0 +1,113 @@
package proxy
import (
"fmt"
"mime"
"net/http"
"path"
"strconv"
"strings"
"time"
)
// CacheControl sets Cache-Control response header with different ages for different mimes
type CacheControl struct {
defaultMaxAge time.Duration
maxAges map[string]time.Duration
}
// NewCacheControl creates NewCacheControl with the default max age
func NewCacheControl(defaultAge time.Duration) *CacheControl {
return &CacheControl{defaultMaxAge: defaultAge, maxAges: map[string]time.Duration{}}
}
// AddMime sets max age for a given mime
func (c *CacheControl) AddMime(m string, d time.Duration) {
c.maxAges[m] = d
}
// Middleware checks if mime custom age set and returns it if matched to content type from resource (file) extension.
// fallback to default if nothing matched
func (c *CacheControl) Middleware(next http.Handler) http.Handler {
setMaxAgeHeader := func(age time.Duration, w http.ResponseWriter) {
w.Header().Set("Cache-Control", "public, max-age="+strconv.Itoa(int(age.Seconds())))
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if len(c.maxAges) == 0 && c.defaultMaxAge == 0 { // cache control disabled
next.ServeHTTP(w, r)
return
}
if len(c.maxAges) == 0 && c.defaultMaxAge > 0 {
setMaxAgeHeader(c.defaultMaxAge, w)
next.ServeHTTP(w, r)
}
ext := path.Ext(r.URL.Path) // the extension ext should begin with a leading dot, as in ".html"
if ext == "" {
ext = ".html"
}
mt := mime.TypeByExtension(ext)
if elems := strings.Split(mt, ";"); len(elems) > 1 { // strip suffix after ";", i.e. text/html; charset=utf-8
mt = strings.TrimSpace(elems[0])
}
val := c.defaultMaxAge
if v, ok := c.maxAges[mt]; ok {
val = v
}
setMaxAgeHeader(val, w)
next.ServeHTTP(w, r)
})
}
// MakeCacheControl creates CacheControl from the list of params.
// the first param represents default age and can be just a duration string (i.e. 60h) or "default:60h"
// all other params are mime:duration pairs, i.e. "text/html:30s"
func MakeCacheControl(cacheOpts []string) (*CacheControl, error) {
if len(cacheOpts) == 0 {
return NewCacheControl(0), nil
}
res := NewCacheControl(0)
// first elements may define default in both "10s" and "default:10s" forms
if !strings.Contains(cacheOpts[0], ":") { // single element, i.e 10s
dur, err := time.ParseDuration(cacheOpts[0])
if err != nil {
return nil, fmt.Errorf("can't parse default cache duration: %w", err)
}
res = NewCacheControl(dur)
}
if strings.Contains(cacheOpts[0], ":") { // two elements, i.e default:10s
elems := strings.Split(cacheOpts[0], ":")
if elems[0] != "default" {
return nil, fmt.Errorf("first cache duration has to be for the default mime")
}
dur, err := time.ParseDuration(elems[1])
if err != nil {
return nil, fmt.Errorf("can't parse default cache duration: %w", err)
}
res = NewCacheControl(dur)
}
// default only, no mime types
if len(cacheOpts) == 1 {
return res, nil
}
for _, v := range cacheOpts[1:] {
elems := strings.Split(v, ":")
if len(elems) != 2 {
return nil, fmt.Errorf("invalid mime:age entry %q", v)
}
dur, err := time.ParseDuration(elems[1])
if err != nil {
return nil, fmt.Errorf("can't parse cache duration from %s: %w", v, err)
}
res.AddMime(elems[0], dur)
}
return res, nil
}

View File

@@ -0,0 +1,125 @@
package proxy
import (
"errors"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCacheControl_MiddlewareDefault(t *testing.T) {
req := httptest.NewRequest("GET", "/file.html", nil)
w := httptest.NewRecorder()
h := NewCacheControl(time.Hour).Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("something"))
}))
h.ServeHTTP(w, req)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "public, max-age=3600", resp.Header.Get("Cache-Control"))
}
func TestCacheControl_MiddlewareDisabled(t *testing.T) {
req := httptest.NewRequest("GET", "/file.html", nil)
w := httptest.NewRecorder()
h := NewCacheControl(0).Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("something"))
}))
h.ServeHTTP(w, req)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "", resp.Header.Get("Cache-Control"))
}
func TestCacheControl_MiddlewareMime(t *testing.T) {
cc := NewCacheControl(time.Hour)
cc.AddMime("text/html", time.Hour*2)
cc.AddMime("image/png", time.Hour*10)
h := cc.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("something"))
}))
{
req := httptest.NewRequest("GET", "/file.html", nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "public, max-age=7200", resp.Header.Get("Cache-Control"), "match on .html")
}
{
req := httptest.NewRequest("GET", "/xyz/file.png?something=blah", nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "public, max-age=36000", resp.Header.Get("Cache-Control"), "match on png")
}
{
req := httptest.NewRequest("GET", "/xyz/file.gif?something=blah", nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "public, max-age=3600", resp.Header.Get("Cache-Control"), "no match, default")
}
{
req := httptest.NewRequest("GET", "/xyz/", nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "public, max-age=7200", resp.Header.Get("Cache-Control"), "match on empty (index)")
}
}
func TestMakeCacheControl(t *testing.T) {
tbl := []struct {
opts []string
defAge time.Duration
mimeAges map[string]time.Duration
err error
}{
{nil, time.Duration(0), nil, nil},
{[]string{"12h"}, 12 * time.Hour, nil, nil},
{[]string{"default:12h"}, 12 * time.Hour, nil, nil},
{[]string{"blah:12h"}, 0, nil, errors.New("first cache duration has to be for the default mime")},
{[]string{"a12bad"}, 0, nil, errors.New(`can't parse default cache duration: time: invalid duration "a12bad"`)},
{[]string{"default:a12bad"}, 0, nil, errors.New(`can't parse default cache duration: time: invalid duration "a12bad"`)},
{[]string{"12h", "text/html:10h", "image/png:6h"}, 12 * time.Hour,
map[string]time.Duration{"text/html": 10 * time.Hour, "image/png": 6 * time.Hour}, nil},
{[]string{"12h", "10h", "image/png:6h"}, 0, nil, errors.New(`invalid mime:age entry "10h"`)},
{[]string{"12h", "abc:10zzh", "image/png:6h"}, 0, nil,
errors.New(`can't parse cache duration from abc:10zzh: time: unknown unit "zzh" in duration "10zzh"`)},
}
for i, tt := range tbl {
t.Run(strconv.Itoa(i), func(t *testing.T) {
res, err := MakeCacheControl(tt.opts)
if tt.err != nil {
require.EqualError(t, err, tt.err.Error())
return
}
require.NoError(t, err)
assert.Equal(t, tt.defAge, res.defaultMaxAge)
for mime, age := range tt.mimeAges {
assert.Equal(t, age, res.maxAges[mime])
}
assert.Equal(t, len(tt.mimeAges), len(res.maxAges))
})
}
}

View File

@@ -24,20 +24,20 @@ import (
// Http is a proxy server for both http and https
type Http struct { // nolint golint
Matcher
Address string
AssetsLocation string
AssetsWebRoot string
AssetsCacheDuration time.Duration
MaxBodySize int64
GzEnabled bool
ProxyHeaders []string
SSLConfig SSLConfig
Version string
AccessLog io.Writer
StdOutEnabled bool
Signature bool
Timeouts Timeouts
Metrics Metrics
Address string
AssetsLocation string
AssetsWebRoot string
MaxBodySize int64
GzEnabled bool
ProxyHeaders []string
SSLConfig SSLConfig
Version string
AccessLog io.Writer
StdOutEnabled bool
Signature bool
Timeouts Timeouts
CacheControl MiddlewareProvider
Metrics MiddlewareProvider
}
// Matcher source info (server and route) to the destination url
@@ -48,8 +48,8 @@ type Matcher interface {
Mappers() (mappers []discovery.URLMapper)
}
// Metrics wraps middleware publishing counts
type Metrics interface {
// MiddlewareProvider interface defines http middleware handler
type MiddlewareProvider interface {
Middleware(next http.Handler) http.Handler
}
@@ -190,7 +190,7 @@ func (h *Http) proxyHandler() http.HandlerFunc {
if h.AssetsLocation != "" && h.AssetsWebRoot != "" {
fs, err := R.FileServer(h.AssetsWebRoot, h.AssetsLocation)
if err == nil {
assetsHandler = h.cachingHandler(fs).ServeHTTP
assetsHandler = h.CacheControl.Middleware(fs).ServeHTTP
}
}
@@ -228,7 +228,7 @@ func (h *Http) proxyHandler() http.HandlerFunc {
http.Error(w, "Server error", http.StatusInternalServerError)
return
}
h.cachingHandler(fs).ServeHTTP(w, r)
h.CacheControl.Middleware(fs).ServeHTTP(w, r)
}
}
}
@@ -310,15 +310,17 @@ func (h *Http) stdoutLogHandler(enable bool, lh func(next http.Handler) http.Han
}
}
func (h *Http) cachingHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if h.AssetsCacheDuration > 0 {
w.Header().Set("Cache-Control", "public, max-age="+strconv.Itoa(int(h.AssetsCacheDuration.Seconds())))
}
next.ServeHTTP(w, r)
})
}
// func (h *Http) cachingHandler(next http.Handler) http.Handler {
//
// return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// mt := mime.TypeByExtension(path.Ext(r.URL.Path))
// log.Printf("tt: %s", mt)
// if h.AssetsCacheDuration > 0 {
// w.Header().Set("Cache-Control", "public, max-age="+strconv.Itoa(int(h.AssetsCacheDuration.Seconds())))
// }
// next.ServeHTTP(w, r)
// })
// }
func (h *Http) makeHTTPServer(addr string, router http.Handler) *http.Server {
return &http.Server{

View File

@@ -4,17 +4,13 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"net/http/httptest"
"os"
"path"
"strconv"
"testing"
"time"
R "github.com/go-pkgz/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -102,8 +98,9 @@ func TestHttp_Do(t *testing.T) {
func TestHttp_DoWithAssets(t *testing.T) {
port := rand.Intn(10000) + 40000
cc := NewCacheControl(time.Hour * 12)
h := Http{Timeouts: Timeouts{ResponseHeader: 200 * time.Millisecond}, Address: fmt.Sprintf("127.0.0.1:%d", port),
AccessLog: io.Discard, AssetsWebRoot: "/static", AssetsLocation: "testdata"}
AccessLog: io.Discard, AssetsWebRoot: "/static", AssetsLocation: "testdata", CacheControl: cc}
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@@ -163,14 +160,16 @@ func TestHttp_DoWithAssets(t *testing.T) {
assert.Equal(t, "test html", string(body))
assert.Equal(t, "", resp.Header.Get("App-Name"))
assert.Equal(t, "", resp.Header.Get("h1"))
assert.Equal(t, "public, max-age=43200", resp.Header.Get("Cache-Control"))
}
}
func TestHttp_DoWithAssetRules(t *testing.T) {
port := rand.Intn(10000) + 40000
cc := NewCacheControl(time.Hour * 12)
h := Http{Timeouts: Timeouts{ResponseHeader: 200 * time.Millisecond}, Address: fmt.Sprintf("127.0.0.1:%d", port),
AccessLog: io.Discard}
AccessLog: io.Discard, CacheControl: cc}
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@@ -231,6 +230,7 @@ func TestHttp_DoWithAssetRules(t *testing.T) {
assert.Equal(t, "test html", string(body))
assert.Equal(t, "", resp.Header.Get("App-Name"))
assert.Equal(t, "", resp.Header.Get("h1"))
assert.Equal(t, "public, max-age=43200", resp.Header.Get("Cache-Control"))
}
}
@@ -257,47 +257,47 @@ func TestHttp_toHttp(t *testing.T) {
}
func TestHttp_cachingHandler(t *testing.T) {
dir, e := ioutil.TempDir(os.TempDir(), "reproxy")
require.NoError(t, e)
e = ioutil.WriteFile(path.Join(dir, "1.html"), []byte("1.htm"), 0600)
assert.NoError(t, e)
e = ioutil.WriteFile(path.Join(dir, "2.html"), []byte("2.htm"), 0600)
assert.NoError(t, e)
defer os.RemoveAll(dir)
fh, e := R.FileServer("/static", dir)
require.NoError(t, e)
h := Http{AssetsCacheDuration: 10 * time.Second, AssetsLocation: dir, AssetsWebRoot: "/static"}
hh := R.Wrap(fh, h.cachingHandler)
ts := httptest.NewServer(hh)
defer ts.Close()
client := http.Client{Timeout: 599 * time.Second}
{
resp, err := client.Get(ts.URL + "/static/1.html")
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
t.Logf("headers: %+v", resp.Header)
assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control"))
assert.NotEqual(t, "", resp.Header.Get("Last-Modified"))
}
{
resp, err := client.Get(ts.URL + "/static/bad.html")
require.NoError(t, err)
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
t.Logf("headers: %+v", resp.Header)
assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control"))
assert.Equal(t, "", resp.Header.Get("Last-Modified"))
}
{
resp, err := client.Get(ts.URL + "/%2e%2e%2f%2e%2e%2f%2e%2e%2f/etc/passwd")
require.NoError(t, err)
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
t.Logf("headers: %+v", resp.Header)
assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control"))
assert.Equal(t, "", resp.Header.Get("Last-Modified"))
}
}
// func TestHttp_cachingHandler(t *testing.T) {
//
// dir, e := ioutil.TempDir(os.TempDir(), "reproxy")
// require.NoError(t, e)
// e = ioutil.WriteFile(path.Join(dir, "1.html"), []byte("1.htm"), 0600)
// assert.NoError(t, e)
// e = ioutil.WriteFile(path.Join(dir, "2.html"), []byte("2.htm"), 0600)
// assert.NoError(t, e)
//
// defer os.RemoveAll(dir)
//
// fh, e := R.FileServer("/static", dir)
// require.NoError(t, e)
// h := Http{AssetsCacheDuration: 10 * time.Second, AssetsLocation: dir, AssetsWebRoot: "/static"}
// hh := R.Wrap(fh, h.cachingHandler)
// ts := httptest.NewServer(hh)
// defer ts.Close()
// client := http.Client{Timeout: 599 * time.Second}
//
// {
// resp, err := client.Get(ts.URL + "/static/1.html")
// require.NoError(t, err)
// assert.Equal(t, http.StatusOK, resp.StatusCode)
// t.Logf("headers: %+v", resp.Header)
// assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control"))
// assert.NotEqual(t, "", resp.Header.Get("Last-Modified"))
// }
// {
// resp, err := client.Get(ts.URL + "/static/bad.html")
// require.NoError(t, err)
// assert.Equal(t, http.StatusNotFound, resp.StatusCode)
// t.Logf("headers: %+v", resp.Header)
// assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control"))
// assert.Equal(t, "", resp.Header.Get("Last-Modified"))
// }
// {
// resp, err := client.Get(ts.URL + "/%2e%2e%2f%2e%2e%2f%2e%2e%2f/etc/passwd")
// require.NoError(t, err)
// assert.Equal(t, http.StatusNotFound, resp.StatusCode)
// t.Logf("headers: %+v", resp.Header)
// assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control"))
// assert.Equal(t, "", resp.Header.Get("Last-Modified"))
// }
// }