From 401375b8d78ed0cedd22de72b59ecfaa18f900e5 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 17 Apr 2021 12:46:22 -0500 Subject: [PATCH 1/6] bump go-pkgz/rest to show hostname in stdout logs --- go.mod | 2 +- go.sum | 2 ++ vendor/github.com/go-pkgz/rest/logger/logger.go | 11 +++++++++-- vendor/modules.txt | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 0513c25..1fe7dda 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.16 require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/go-pkgz/lgr v0.10.4 - github.com/go-pkgz/rest v1.9.1 + github.com/go-pkgz/rest v1.9.2 github.com/gorilla/handlers v1.5.1 github.com/stretchr/testify v1.7.0 github.com/umputun/go-flags v1.5.1 diff --git a/go.sum b/go.sum index c020179..6b8d66d 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw= github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0= github.com/go-pkgz/rest v1.9.1 h1:JW876BgJJ/MOkAYRnnzpfX7xUqIav+ou1LSVTtQq/Lo= github.com/go-pkgz/rest v1.9.1/go.mod h1:wZ/dGipZUaF9to0vIQl7PwDHgWQDB0jsrFg1xnAKLDw= +github.com/go-pkgz/rest v1.9.2 h1:RyBBRXBYY6eBgTW3UGYOyT4VQPDiBBFh/tesELWsryQ= +github.com/go-pkgz/rest v1.9.2/go.mod h1:wZ/dGipZUaF9to0vIQl7PwDHgWQDB0jsrFg1xnAKLDw= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/vendor/github.com/go-pkgz/rest/logger/logger.go b/vendor/github.com/go-pkgz/rest/logger/logger.go index 0efbd9a..a95d592 100644 --- a/vendor/github.com/go-pkgz/rest/logger/logger.go +++ b/vendor/github.com/go-pkgz/rest/logger/logger.go @@ -41,6 +41,7 @@ type logParts struct { remoteIP string statusCode int respSize int + host string prefix string user string @@ -108,10 +109,16 @@ func (l *Middleware) Handler(next http.Handler) http.Handler { remoteIP = l.ipFn(remoteIP) } + server := r.URL.Hostname() + if server == "" { + server = strings.Split(r.Host, ":")[0] + } + p := &logParts{ duration: t2.Sub(t1), rawURL: rawurl, method: r.Method, + host: server, remoteIP: remoteIP, statusCode: ww.status, respSize: ww.size, @@ -135,8 +142,8 @@ func (l *Middleware) formatDefault(r *http.Request, p *logParts) string { _, _ = bld.WriteString(" ") } - _, _ = bld.WriteString(fmt.Sprintf("%s - %s - %s - %d (%d) - %v", - p.method, p.rawURL, p.remoteIP, p.statusCode, p.respSize, p.duration)) + _, _ = bld.WriteString(fmt.Sprintf("%s - %s - %s - %s - %d (%d) - %v", + p.method, p.rawURL, p.host, p.remoteIP, p.statusCode, p.respSize, p.duration)) if p.user != "" { _, _ = bld.WriteString(" - ") diff --git a/vendor/modules.txt b/vendor/modules.txt index b554a01..e28187e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,7 +7,7 @@ github.com/felixge/httpsnoop # github.com/go-pkgz/lgr v0.10.4 ## explicit github.com/go-pkgz/lgr -# github.com/go-pkgz/rest v1.9.1 +# github.com/go-pkgz/rest v1.9.2 ## explicit github.com/go-pkgz/rest github.com/go-pkgz/rest/logger From 34c1510c8cda37559652dc22fe2c5f34700d4310 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 17 Apr 2021 12:49:47 -0500 Subject: [PATCH 2/6] fix docker paths in docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a1609e1..0959df5 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Example with an automatic docker discovery: ## Install - for a binary distribution pick the proper file in the [release section](https://github.com/umputun/reproxy/releases) -- docker container available on [Docker Hub](https://hub.docker.com/r/umputun/reproxy) as well as on [Github Container Registry](ghcr.io/umputun/reproxy). +- docker container available on [Docker Hub](https://hub.docker.com/r/umputun/reproxy) as well as on [Github Container Registry](https://ghcr.io/umputun/reproxy). I.e. `docker pull umputun/reproxy` or `docker pull ghcr.io/umputun/reproxy`. Latest stable version has `:vX.Y.Z` tag (with `:latest` alias) and the current master has `:master` tag. From bbbd24dd536c8acd84c3209192eec59c37147d72 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 17 Apr 2021 13:11:10 -0500 Subject: [PATCH 3/6] fix missing url.Host forward --- app/main_test.go | 15 +++++++++++++-- app/proxy/proxy.go | 5 +++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/app/main_test.go b/app/main_test.go index 0ea87b5..aa65140 100644 --- a/app/main_test.go +++ b/app/main_test.go @@ -21,6 +21,7 @@ func Test_Main(t *testing.T) { port := chooseRandomUnusedPort() os.Args = []string{"test", "--static.enabled", "--static.rule=*,/svc1, https://httpbin.org/get,https://feedmaster.umputun.com/ping", + "--static.rule=*,/svc2/(.*), https://echo.umputun.com/$1,https://feedmaster.umputun.com/ping", "--dbg", "--logger.stdout", "--listen=127.0.0.1:" + strconv.Itoa(port), "--signature"} done := make(chan struct{}) @@ -62,11 +63,21 @@ func Test_Main(t *testing.T) { assert.Equal(t, 200, resp.StatusCode) body, err := ioutil.ReadAll(resp.Body) assert.NoError(t, err) - assert.Contains(t, string(body), `"Host": "127.0.0.1"`) + assert.Contains(t, string(body), `"Host": "httpbin.org"`) } { client := http.Client{Timeout: 10 * time.Second} - resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/bas", port)) + resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/svc2/test", port)) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Contains(t, string(body), `echo echo 123`) + } + { + client := http.Client{Timeout: 10 * time.Second} + resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/bad", port)) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusBadGateway, resp.StatusCode) diff --git a/app/proxy/proxy.go b/app/proxy/proxy.go index 3b97998..5de5f18 100644 --- a/app/proxy/proxy.go +++ b/app/proxy/proxy.go @@ -151,11 +151,12 @@ func (h *Http) proxyHandler() http.HandlerFunc { Director: func(r *http.Request) { ctx := r.Context() uu := ctx.Value(contextKey("url")).(*url.URL) + r.Header.Add("X-Forwarded-Host", uu.Host) + r.Header.Set("X-Origin-Host", r.Host) r.URL.Path = uu.Path r.URL.Host = uu.Host r.URL.Scheme = uu.Scheme - r.Header.Add("X-Forwarded-Host", uu.Host) - r.Header.Add("X-Origin-Host", r.Host) + r.Host = uu.Host h.setXRealIP(r) }, Transport: &http.Transport{ From f43e9092df5617354201d71ec3ba4b2109e92704 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 17 Apr 2021 13:17:02 -0500 Subject: [PATCH 4/6] regen site --- site/public/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/public/index.html b/site/public/index.html index 8681d99..fb08dce 100644 --- a/site/public/index.html +++ b/site/public/index.html @@ -1,4 +1,4 @@ -Reproxy

Reproxy is a simple edge HTTP(s) server / reverse proxy supporting various providers (docker, static, file).
One or more providers supply information about requested server, requested url, destination url and health check url.
Distributed as a single binary or as a docker container.

  • Automatic SSL termination with Let's Encrypt
  • Support of user-provided SSL certificates
  • Simple but flexible proxy rules
  • Static, command line proxy rules provider
  • Dynamic, file-based proxy rules provider
  • Docker provider with an automatic discovery
  • Optional traffic compression
  • User-defined limits and timeouts
  • Single binary distribution
  • Docker container distribution
  • Built-in static assets server

build Coverage Status Go Report Card Docker Automated build

Server can be set as FQDN, i.e. s.example.com or * (catch all). Requested url can be regex, for example ^/api/(.*) and destination url may have regex matched groups in, i.e. http://d.example.com:8080/$1. For the example above http://s.example.com/api/something?foo=bar will be proxied to http://d.example.com:8080/something?foo=bar.

For convenience, requests with the trailing / and without regex groups expanded to /(.*), and destinations in those cases expanded to /$1. I.e. /api/ -> http://127.0.0.1/service will be translated to ^/api/(.*) -> http://127.0.0.1/service/$1

Both HTTP and HTTPS supported. For HTTPS, static certificate can be used as well as automated ACME (Let's Encrypt) certificates. Optional assets server can be used to serve static files.

Starting reproxy requires at least one provider defined. The rest of parameters are strictly optional and have sane default.

Example with a static provider:
reproxy --static.enabled --static.rule="example.com/api/(.*),https://api.example.com/$1"
Example with an automatic docker discovery:
reproxy --docker.enabled --docker.auto

Install

Latest stable version has :vX.Y.Z tag (with :latest alias) and the current master has :master tag.

Providers

User can sets multiple providers at the same time.
See examples of various providers in examples

Static

This is the simplest provider defining all mapping rules directly in the command line (or environment). Multiple rules supported.
Each rule is 3 or 4 comma-separated elements server,sourceurl,destination,[ping-url]. For example:

  • *,^/api/(.*),https://api.example.com/$1, - proxy all request to any host/server with /api prefix to https://api.example.com
  • example.com,/foo/bar,https://api.example.com/zzz,https://api.example.com/ping - proxy all requests to example.com and with /foo/bar url to https://api.example.com/zzz. Uses https://api.example.com/ping for the health check

The last (4th) element defines an optional ping url used for health reporting. I.e.*,^/api/(.*),https://api.example.com/$1,https://api.example.com/ping. See Health check section for more details.

File

reproxy --file.enabled --file.name=config.yml

Example of config.yml:

default: # the same as * (catch-all) server
+Reproxy
Reproxy is a simple edge HTTP(s) server / reverse proxy supporting various providers (docker, static, file).
One or more providers supply information about requested server, requested url, destination url and health check url.
Distributed as a single binary or as a docker container.
  • Automatic SSL termination with Let's Encrypt
  • Support of user-provided SSL certificates
  • Simple but flexible proxy rules
  • Static, command line proxy rules provider
  • Dynamic, file-based proxy rules provider
  • Docker provider with an automatic discovery
  • Optional traffic compression
  • User-defined limits and timeouts
  • Single binary distribution
  • Docker container distribution
  • Built-in static assets server
build Coverage Status Go Report Card Docker Automated build
Server can be set as FQDN, i.e. s.example.com or * (catch all). Requested url can be regex, for example ^/api/(.*) and destination url may have regex matched groups in, i.e. http://d.example.com:8080/$1. For the example above http://s.example.com/api/something?foo=bar will be proxied to http://d.example.com:8080/something?foo=bar.
For convenience, requests with the trailing / and without regex groups expanded to /(.*), and destinations in those cases expanded to /$1. I.e. /api/ -> http://127.0.0.1/service will be translated to ^/api/(.*) -> http://127.0.0.1/service/$1
Both HTTP and HTTPS supported. For HTTPS, static certificate can be used as well as automated ACME (Let's Encrypt) certificates. Optional assets server can be used to serve static files.
Starting reproxy requires at least one provider defined. The rest of parameters are strictly optional and have sane default.
Example with a static provider:
reproxy --static.enabled --static.rule="example.com/api/(.*),https://api.example.com/$1"
Example with an automatic docker discovery:
reproxy --docker.enabled --docker.auto
Install 
Latest stable version has :vX.Y.Z tag (with :latest alias) and the current master has :master tag.
Providers 
Proxy rules supplied by various providers. Currently included file, docker and static. Each provider may define multiple routing rules for both proxied request and static (assets). User can sets multiple providers at the same time.
See examples of various providers in examples
Static 
This is the simplest provider defining all mapping rules directly in the command line (or environment). Multiple rules supported.
Each rule is 3 or 4 comma-separated elements server,sourceurl,destination,[ping-url]. For example:
  • *,^/api/(.*),https://api.example.com/$1, - proxy all request to any host/server with /api prefix to https://api.example.com
  • example.com,/foo/bar,https://api.example.com/zzz,https://api.example.com/ping - proxy all requests to example.com and with /foo/bar url to https://api.example.com/zzz. Uses https://api.example.com/ping for the health check
The last (4th) element defines an optional ping url used for health reporting. I.e.*,^/api/(.*),https://api.example.com/$1,https://api.example.com/ping. See Health check section for more details.
File 
reproxy --file.enabled --file.name=config.yml
Example of config.yml:
default: # the same as * (catch-all) server
   - { route: "^/api/svc1/(.*)", dest: "http://127.0.0.1:8080/blah1/$1" }
   - {
       route: "/api/svc3/xyz",
@@ -67,7 +67,7 @@ timeout:
 Help Options:
   -h, --help                        Show this help message
 
-
Status 
The project is under active development and may have breaking changes till v1 released.
Updated  Edit