fix some errors & add CLI options & extend readme

acme.go

@@ -4,6 +4,7 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 
@@ -84,6 +85,10 @@ func dump(config *Config, data *StoredData) error {
 		}
 	}
 
+	if config.Watch {
+		log.Println("wrote new configuration")
+	}
+
 	if err := tree(config.Path, ""); err != nil {
 		return err
 	}

dumper.go

@@ -1,7 +1,5 @@
 package main
 
-import "fmt"
-
 const (
 	// FILE backend
 	FILE string = "file"
@@ -9,8 +7,8 @@ const (
 	CONSUL string = "consul"
 	// ETCD backend
 	ETCD string = "etcd"
-	// ZK backend
+	// ZOOKEEPER backend
-	ZK string = "zk"
+	ZOOKEEPER string = "zookeeper"
 	// BOLTDB backend
 	BOLTDB string = "boltdb"
 )
@@ -27,15 +25,14 @@ type Config struct {
 
 // Backend represents an object storage of ACME data
 type Backend interface {
-	loop(watch bool) (<-chan *StoredData, <-chan error)
+	getStoredData(watch bool) (<-chan *StoredData, <-chan error)
 }
 
 func run(config *Config) error {
-	data, errors := config.BackendConfig.(Backend).loop(config.Watch)
+	data, errors := config.BackendConfig.(Backend).getStoredData(config.Watch)
 	for {
 		select {
 		case err := <-errors:
-			fmt.Println(err)
 			return err
 		case acmeData, ok := <-data:
 			if !ok {

file.go

@@ -33,7 +33,28 @@ func sendStoredData(path string, dataCh chan *StoredData, errCh chan error) {
 	dataCh <- data
 }
 
-func (b FileBackend) loop(watch bool) (<-chan *StoredData, <-chan error) {
+func loopFile(path string, watcher *fsnotify.Watcher, dataCh chan *StoredData, errCh chan error) {
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Op&fsnotify.Write == fsnotify.Write {
+					sendStoredData(path, dataCh, errCh)
+				}
+			case err1, ok := <-watcher.Errors:
+				if !ok {
+					return
+				}
+				errCh <- err1
+			}
+		}
+	}()
+}
+
+func (b FileBackend) getStoredData(watch bool) (<-chan *StoredData, <-chan error) {
 
 	dataCh := make(chan *StoredData)
 	errCh := make(chan error)
@@ -54,24 +75,7 @@ func (b FileBackend) loop(watch bool) (<-chan *StoredData, <-chan error) {
 		errCh <- err
 	}
 
-	go func() {
+	go loopFile(b.Path, watcher, dataCh, errCh)
-		for {
-			select {
-			case event, ok := <-watcher.Events:
-				if !ok {
-					return
-				}
-				if event.Op&fsnotify.Write == fsnotify.Write {
-					sendStoredData(b.Path, dataCh, errCh)
-				}
-			case err1, ok := <-watcher.Errors:
-				if !ok {
-					return
-				}
-				errCh <- err1
-			}
-		}
-	}()
 
 	err = watcher.Add(b.Path)
 	if err != nil {

kv.go

@@ -55,7 +55,7 @@ func register(backend string) (store.Backend, error) {
 	case ETCD:
 		etcdv3.Register()
 		return store.ETCDV3, nil
-	case ZK:
+	case ZOOKEEPER:
 		zookeeper.Register()
 		return store.ZK, nil
 	case BOLTDB:
@@ -66,42 +66,81 @@ func register(backend string) (store.Backend, error) {
 	}
 }
 
-func (b KVBackend) loop(watch bool) (<-chan *StoredData, <-chan error) {
+func loopKV(watch bool, kvstore store.Store, dataCh chan *StoredData, errCh chan error) {
+	stopCh := make(<-chan struct{})
+	events, err := kvstore.Watch(storeKey, stopCh, nil)
+	if err != nil {
+		errCh <- err
+	}
+	for {
+		kvpair := <-events
+		if kvpair == nil {
+			errCh <- fmt.Errorf("could not fetch Key/Value pair for key %v", storeKey)
+			return
+		}
+		dataCh <- extractStoredData(kvpair, errCh)
+		if !watch {
+			close(dataCh)
+			close(errCh)
+		}
+	}
+}
+
+func extractStoredData(kvpair *store.KVPair, errCh chan error) *StoredData {
+	storedData, err := getStoredDataFromGzip(kvpair.Value)
+	if err != nil {
+		errCh <- err
+	}
+	return storedData
+}
+
+func getSingleData(kvstore store.Store, dataCh chan *StoredData, errCh chan error) {
+	kvpair, err := kvstore.Get(storeKey, nil)
+	if err != nil {
+		errCh <- err
+		return
+	}
+	if kvpair == nil {
+		errCh <- fmt.Errorf("could not fetch Key/Value pair for key %v", storeKey)
+		return
+	}
+	dataCh <- extractStoredData(kvpair, errCh)
+	close(dataCh)
+	close(errCh)
+}
+
+func (b KVBackend) getStoredData(watch bool) (<-chan *StoredData, <-chan error) {
 
 	dataCh := make(chan *StoredData)
-	errors := make(chan error)
+	errCh := make(chan error)
 
 	backend, err := register(b.Name)
 	if err != nil {
-		errors <- err
+		go func() {
+			errCh <- err
+		}()
+		return dataCh, errCh
 	}
-
 	kvstore, err := valkeyrie.NewStore(
 		backend,
 		b.Client,
 		b.Config,
 	)
-	if err != nil {
-		errors <- err
-	}
 
+	if err != nil {
 		go func() {
-		stopCh := make(<-chan struct{})
+			errCh <- err
-		events, _ := kvstore.Watch(storeKey, stopCh, nil)
-		for {
-			kvpair := <-events
-			storedData, err := getStoredDataFromGzip(kvpair.Value)
-			if err != nil {
-				errors <- err
-			}
-			dataCh <- storedData
-			if !watch {
-				close(dataCh)
-				close(errors)
-			}
-		}
 		}()
+		return dataCh, errCh
+	}
 
-	return dataCh, errors
+	if !watch {
+		go getSingleData(kvstore, dataCh, errCh)
+		return dataCh, errCh
+	}
+
+	go loopKV(watch, kvstore, dataCh, errCh)
+
+	return dataCh, errCh
 
 }

main.go

@@ -1,9 +1,15 @@
 package main
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
+	"log"
 	"os"
 	"strconv"
+	"strings"
+	"time"
 
 	"github.com/abronan/valkeyrie/store"
 	"github.com/spf13/cobra"
@@ -13,7 +19,7 @@ func main() {
 	var rootCmd = &cobra.Command{
 		Use:     "traefik-certs-dumper",
 		Short:   "Dump Let's Encrypt certificates from Traefik",
-		Long:    `Dump the content of the "acme.json" file from Traefik to certificates.`,
+		Long:    `Dump ACME data from Traefik of different storage backends to certificates.`,
 		Version: version,
 	}
 
@@ -22,15 +28,18 @@ func main() {
 	var dumpCmd = &cobra.Command{
 		Use:   "dump",
 		Short: "Dump Let's Encrypt certificates from Traefik",
-		Long:  `Dump the content of the "acme.json" file from Traefik to certificates.`,
+		Long:  `Dump ACME data from Traefik of different storage backends to certificates.`,
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			source := cmd.Flag("source").Value.String()
-			sourceFile := cmd.Flag("file").Value.String()
+			sourceFile := cmd.Flag("source.file").Value.String()
-			if source == "file" {
+			watch, _ := strconv.ParseBool(cmd.Flag("watch").Value.String())
+			if source == FILE {
 				if _, err := os.Stat(sourceFile); os.IsNotExist(err) {
-					return fmt.Errorf("--file (%q) does not exist", sourceFile)
+					return fmt.Errorf("--source.file (%q) does not exist", sourceFile)
 				}
-			} else if source != "consul" && source != "etcd" && source != "zookeeper" && source != "boltdb" {
+			} else if source == BOLTDB && watch {
+				return fmt.Errorf("--watch=true is not supported for boltdb")
+			} else if source != CONSUL && source != ETCD && source != ZOOKEEPER && source != BOLTDB {
 				return fmt.Errorf("--source (%q) is not allowed, use one of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'", source)
 			}
 
@@ -49,7 +58,57 @@ func main() {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 
 			source := cmd.Flag("source").Value.String()
-			acmeFile := cmd.Flag("file").Value.String()
+			acmeFile := cmd.Flag("source.file").Value.String()
+
+			endpoints := strings.Split(cmd.Flag("source.kv.endpoints").Value.String(), ",")
+
+			storeConfig := &store.Config{}
+
+			timeout, _ := strconv.Atoi(cmd.Flag("source.kv.connection-timeout").Value.String())
+			storeConfig.ConnectionTimeout = time.Second * time.Duration(timeout)
+			storeConfig.Username = cmd.Flag("source.kv.username").Value.String()
+			storeConfig.Password = cmd.Flag("source.kv.password").Value.String()
+
+			enableTLS, err := strconv.ParseBool(cmd.Flag("source.kv.tls.enable").Value.String())
+			if err != nil {
+				return err
+			}
+
+			if enableTLS {
+				tlsConfig := &tls.Config{}
+				insecureSkipVerify, err := strconv.ParseBool(cmd.Flag("source.kv.tls.insecureskipverify").Value.String())
+				if err != nil {
+					return err
+				}
+				tlsConfig.InsecureSkipVerify = insecureSkipVerify
+				if cmd.Flag("source.kv.tls.ca-cert-file").Value.String() != "" {
+					caFile := cmd.Flag("source.kv.tls.ca-cert-file").Value.String()
+					caCert, err := ioutil.ReadFile(caFile)
+					if err != nil {
+						log.Fatal(err)
+					}
+					roots := x509.NewCertPool()
+					ok := roots.AppendCertsFromPEM(caCert)
+					if !ok {
+						log.Fatalf("failed to parse root certificate")
+					}
+					tlsConfig.RootCAs = roots
+				}
+				storeConfig.TLS = tlsConfig
+			}
+
+			// Special parameters for etcd
+			timeout, _ = strconv.Atoi(cmd.Flag("source.kv.etcd.sync-period").Value.String())
+			storeConfig.SyncPeriod = time.Second * time.Duration(timeout)
+			// Special parameters for boltdb
+			persistConnection, err := strconv.ParseBool(cmd.Flag("source.kv.boltdb.persist-connection").Value.String())
+			if err != nil {
+				return err
+			}
+			storeConfig.PersistConnection = persistConnection
+			storeConfig.Bucket = cmd.Flag("source.kv.boltdb.bucket").Value.String()
+			// Special parameters for consul
+			storeConfig.Token = cmd.Flag("source.kv.consul.token").Value.String()
 
 			switch source {
 			case "file":
@@ -58,28 +117,18 @@ func main() {
 					Path: acmeFile,
 				}
 			case "consul":
-				config.BackendConfig = KVBackend{
+				fallthrough
-					Name:   CONSUL,
-					Client: []string{"localhost:8500"},
-					Config: &store.Config{},
-				}
 			case "etcd":
-				config.BackendConfig = KVBackend{
+				fallthrough
-					Name:   ETCD,
-					Client: []string{"localhost:8500"},
-					Config: &store.Config{},
-				}
 			case "zookeeper":
-				config.BackendConfig = KVBackend{
+				fallthrough
-					Name:   ZK,
-					Client: []string{"localhost:8500"},
-					Config: &store.Config{},
-				}
 			case "boltdb":
+				fallthrough
+			default:
 				config.BackendConfig = KVBackend{
-					Name:   BOLTDB,
+					Name:   source,
-					Client: []string{"localhost:8500"},
+					Client: endpoints,
-					Config: &store.Config{},
+					Config: storeConfig,
 				}
 			}
 
@@ -106,24 +155,26 @@ func main() {
 		},
 	}
 
-	dumpCmd.Flags().String("source", "file", "Source type. One of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'.")
+	dumpCmd.Flags().String("source", "file", "Source type, one of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'. Options for each source type are prefixed with `source.<type>.`")
-	dumpCmd.Flags().String("file", "./acme.json", "Path to 'acme.json' file if source type is 'file'")
+	dumpCmd.Flags().String("source.file", "./acme.json", "Path to 'acme.json' for file source.")
 
-	/* TODO implement this
+	// Generic parameters for Key/Value backends
-	dumpCmd.Flags().String("kv.client")
+	dumpCmd.Flags().String("source.kv.endpoints", "localhost:8500", "Comma seperated list of endpoints.")
-	dumpCmd.Flags().String("kv.connection-timeout")
+	dumpCmd.Flags().Int("source.kv.connection-timeout", 0, "Connection timeout in seconds.")
-	dumpCmd.Flags().String("kv.sync-period")
+	dumpCmd.Flags().String("source.kv.password", "", "Password for connection.")
-	dumpCmd.Flags().String("kv.bucket")
+	dumpCmd.Flags().String("source.kv.username", "", "Username for connection.")
-	dumpCmd.Flags().Bool("kv.persist-connection")
+	dumpCmd.Flags().Bool("source.kv.tls.enable", false, "Enable TLS encryption.")
-	dumpCmd.Flags().String("kv.username")
+	dumpCmd.Flags().Bool("source.kv.tls.insecureskipverify", false, "Trust unverified certificates if TLS is enabled.")
-	dumpCmd.Flags().String("kv.password")
+	dumpCmd.Flags().String("source.kv.tls.ca-cert-file", "", "Root CA file for certificate verification if TLS is enabled.")
-	dumpCmd.Flags().String("kv.token")
+	// Special parameters for etcd
-	dumpCmd.Flags().String("kv.tls-cert-file")
+	dumpCmd.Flags().Int("source.kv.etcd.sync-period", 0, "Sync period for etcd in seconds.")
-	dumpCmd.Flags().String("kv.tls-key-file")
+	// Special parameters for boltdb
-	dumpCmd.Flags().String("kv.tls-ca-cert-file")
+	dumpCmd.Flags().Bool("source.kv.boltdb.persist-connection", false, "Persist connection for boltdb.")
-	*/
+	dumpCmd.Flags().String("source.kv.boltdb.bucket", "traefik", "Bucket for boltdb.")
+	// Special parameters for consul
+	dumpCmd.Flags().String("source.kv.consul.token", "", "Token for consul.")
 
-	dumpCmd.Flags().Bool("watch", true, "Enable watching changes.")
+	dumpCmd.Flags().Bool("watch", false, "Enable watching changes.")
 	dumpCmd.Flags().String("dest", "./dump", "Path to store the dump content.")
 	dumpCmd.Flags().String("crt-ext", ".crt", "The file extension of the generated certificates.")
 	dumpCmd.Flags().String("crt-name", "certificate", "The file name (without extension) of the generated certificates.")

readme.md

@@ -44,7 +44,7 @@ docker run ldez/traefik-certs-dumper:<tag_name>
 ## Usage
 
 ```yaml
-Dump the content of the "acme.json" file from Traefik to certificates.
+Dump ACME data from Traefik of different storage backends to certificates.
 
 Usage:
   traefik-certs-dumper [command]
@@ -55,14 +55,14 @@ Available Commands:
   version     Display version
 
 Flags:
-  -h, --help      help for certs-dumper
+  -h, --help      help for traefik-certs-dumper
-      --version   version for certs-dumper
+      --version   version for traefik-certs-dumper
 
 Use "traefik-certs-dumper [command] --help" for more information about a command.
 ```
 
 ```yaml
-Dump the content of the "acme.json" file from Traefik to certificates.
+Dump ACME data from Traefik of different storage backends to certificates.
 
 Usage:
   traefik-certs-dumper dump [flags]
@@ -75,7 +75,20 @@ Flags:
   -h, --help                                  help for dump
       --key-ext string                        The file extension of the generated private keys. (default ".key")
       --key-name string                       The file name (without extension) of the generated private keys. (default "privatekey")
-      --source string     Path to 'acme.json' file. (default "./acme.json")
+      --source source.<type>.                 Source type, one of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'. Options for each source type are prefixed with source.<type>. (default "file")
+      --source.file string                    Path to 'acme.json' for file source. (default "./acme.json")
+      --source.kv.boltdb.bucket string        Bucket for boltdb. (default "traefik")
+      --source.kv.boltdb.persist-connection   Persist connection for boltdb.
+      --source.kv.connection-timeout int      Connection timeout in seconds.
+      --source.kv.consul.token string         Token for consul.
+      --source.kv.endpoints string            Comma seperated list of endpoints. (default "localhost:8500")
+      --source.kv.etcd.sync-period int        Sync period for etcd in seconds.
+      --source.kv.password string             Password for connection.
+      --source.kv.tls.ca-cert-file string     Root CA file for certificate verification if TLS is enabled.
+      --source.kv.tls.enable                  Enable TLS encryption.
+      --source.kv.tls.insecureskipverify      Trust unverified certificates if TLS is enabled.
+      --source.kv.username string             Username for connection.
+      --watch                                 Enable watching changes.
 ```
 
 ## Examples
@@ -93,6 +106,51 @@ dump
 
 ```
 
+### Enabled watching
+
+```console
+$ traefik-certs-dumper dump --watch
+2019/04/19 16:56:34 wrote new configuration
+dump
+├──certs
+│  └──my.domain.com.key
+└──private
+   ├──my.domain.com.crt
+   └──letsencrypt.key
+2019/04/19 16:57:14 wrote new configuration
+dump
+├──certs
+│  └──my.domain.com.key
+└──private
+   ├──my.domain.com.crt
+   └──letsencrypt.key
+
+```
+
+### Consul backend
+
+```console
+$ traefik-certs-dumper dump --source consul --source.kv.endpoints=localhost:8500
+```
+
+### Etcd backend
+
+```console
+$ traefik-certs-dumper dump --source etcd --source.kv.endpoints=localhost:2379
+```
+
+### Boltdb backend
+
+```console
+$ traefik-certs-dumper dump --source boltdb --source.kv.endpoints=/tmp/my.db
+```
+
+### Zookeeper backend
+
+```console
+$ traefik-certs-dumper dump --source zookeeper --source.kv.endpoints=localhost:2181
+```
+
 ### Change source and destination
 
 ```console