From 2db90bfccccee93d9320ab97575c96ddbd6335a1 Mon Sep 17 00:00:00 2001
From: Danielle Tomlinson <dani@hashicorp.com>
Date: Tue, 19 Feb 2019 14:38:39 +0100
Subject: [PATCH] dlogger: Increase resilience to docker api failure

This commit adds some extra resiliency to the docker logger in the case
of API failure from the docker daemon, by restarting the stream from the
current point in time if the stream returns and the container is still
running.
---
 drivers/docker/docklog/docker_logger.go | 48 +++++++++++++++++++------
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/drivers/docker/docklog/docker_logger.go b/drivers/docker/docklog/docker_logger.go
index afa1c89c7..55f3e365c 100644
--- a/drivers/docker/docklog/docker_logger.go
+++ b/drivers/docker/docklog/docker_logger.go
@@ -3,6 +3,7 @@ package docklog
 import (
 	"fmt"
 	"io"
+	"time"
 
 	docker "github.com/fsouza/go-dockerclient"
 	hclog "github.com/hashicorp/go-hclog"
@@ -31,6 +32,10 @@ type StartOpts struct {
 	//Stderr path to fifo
 	Stderr string
 
+	// StartTime is the Unix time that the docker logger should fetch logs beginning
+	// from
+	StartTime int64
+
 	// TLS settings for docker client
 	TLSCert string
 	TLSKey  string
@@ -75,18 +80,39 @@ func (d *dockerLogger) Start(opts *StartOpts) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	d.cancelCtx = cancel
 
-	logOpts := docker.LogsOptions{
-		Context:      ctx,
-		Container:    opts.ContainerID,
-		OutputStream: d.stdout,
-		ErrorStream:  d.stderr,
-		Since:        0,
-		Follow:       true,
-		Stdout:       true,
-		Stderr:       true,
-	}
+	go func() {
+		sinceTime := time.Unix(opts.StartTime, 0)
 
-	go func() { client.Logs(logOpts) }()
+		for {
+			logOpts := docker.LogsOptions{
+				Context:      ctx,
+				Container:    opts.ContainerID,
+				OutputStream: d.stdout,
+				ErrorStream:  d.stderr,
+				Since:        sinceTime.Unix(),
+				Follow:       true,
+				Stdout:       true,
+				Stderr:       true,
+			}
+
+			err := client.Logs(logOpts)
+			if err != nil {
+				d.logger.Error("Log streaming ended with error", "error", err)
+			}
+
+			sinceTime = time.Now()
+
+			container, err := client.InspectContainer(opts.ContainerID)
+			if err != nil {
+				_, notFoundOk := err.(*docker.NoSuchContainer)
+				if !notFoundOk {
+					return
+				}
+			} else if container.State.Running != true {
+				return
+			}
+		}
+	}()
 	return nil
 
 }