diff --git a/acme.go b/acme.go index 55ecf2e..70a4381 100644 --- a/acme.go +++ b/acme.go @@ -4,6 +4,7 @@ import ( "encoding/pem" "fmt" "io/ioutil" + "log" "os" "path/filepath" @@ -84,6 +85,10 @@ func dump(config *Config, data *StoredData) error { } } + if config.Watch { + log.Println("wrote new configuration") + } + if err := tree(config.Path, ""); err != nil { return err } diff --git a/dumper.go b/dumper.go index 61b6a11..35af031 100644 --- a/dumper.go +++ b/dumper.go @@ -1,7 +1,5 @@ package main -import "fmt" - const ( // FILE backend FILE string = "file" @@ -9,8 +7,8 @@ const ( CONSUL string = "consul" // ETCD backend ETCD string = "etcd" - // ZK backend - ZK string = "zk" + // ZOOKEEPER backend + ZOOKEEPER string = "zookeeper" // BOLTDB backend BOLTDB string = "boltdb" ) @@ -27,15 +25,14 @@ type Config struct { // Backend represents an object storage of ACME data type Backend interface { - loop(watch bool) (<-chan *StoredData, <-chan error) + getStoredData(watch bool) (<-chan *StoredData, <-chan error) } func run(config *Config) error { - data, errors := config.BackendConfig.(Backend).loop(config.Watch) + data, errors := config.BackendConfig.(Backend).getStoredData(config.Watch) for { select { case err := <-errors: - fmt.Println(err) return err case acmeData, ok := <-data: if !ok { diff --git a/file.go b/file.go index 5ea3208..d5ae59f 100644 --- a/file.go +++ b/file.go @@ -33,7 +33,28 @@ func sendStoredData(path string, dataCh chan *StoredData, errCh chan error) { dataCh <- data } -func (b FileBackend) loop(watch bool) (<-chan *StoredData, <-chan error) { +func loopFile(path string, watcher *fsnotify.Watcher, dataCh chan *StoredData, errCh chan error) { + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + if event.Op&fsnotify.Write == fsnotify.Write { + sendStoredData(path, dataCh, errCh) + } + case err1, ok := <-watcher.Errors: + if !ok { + return + } + errCh <- err1 + } + } + }() +} + +func (b FileBackend) getStoredData(watch bool) (<-chan *StoredData, <-chan error) { dataCh := make(chan *StoredData) errCh := make(chan error) @@ -54,24 +75,7 @@ func (b FileBackend) loop(watch bool) (<-chan *StoredData, <-chan error) { errCh <- err } - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return - } - if event.Op&fsnotify.Write == fsnotify.Write { - sendStoredData(b.Path, dataCh, errCh) - } - case err1, ok := <-watcher.Errors: - if !ok { - return - } - errCh <- err1 - } - } - }() + go loopFile(b.Path, watcher, dataCh, errCh) err = watcher.Add(b.Path) if err != nil { diff --git a/kv.go b/kv.go index c4ec412..dc1f04f 100644 --- a/kv.go +++ b/kv.go @@ -55,7 +55,7 @@ func register(backend string) (store.Backend, error) { case ETCD: etcdv3.Register() return store.ETCDV3, nil - case ZK: + case ZOOKEEPER: zookeeper.Register() return store.ZK, nil case BOLTDB: @@ -66,42 +66,81 @@ func register(backend string) (store.Backend, error) { } } -func (b KVBackend) loop(watch bool) (<-chan *StoredData, <-chan error) { +func loopKV(watch bool, kvstore store.Store, dataCh chan *StoredData, errCh chan error) { + stopCh := make(<-chan struct{}) + events, err := kvstore.Watch(storeKey, stopCh, nil) + if err != nil { + errCh <- err + } + for { + kvpair := <-events + if kvpair == nil { + errCh <- fmt.Errorf("could not fetch Key/Value pair for key %v", storeKey) + return + } + dataCh <- extractStoredData(kvpair, errCh) + if !watch { + close(dataCh) + close(errCh) + } + } +} + +func extractStoredData(kvpair *store.KVPair, errCh chan error) *StoredData { + storedData, err := getStoredDataFromGzip(kvpair.Value) + if err != nil { + errCh <- err + } + return storedData +} + +func getSingleData(kvstore store.Store, dataCh chan *StoredData, errCh chan error) { + kvpair, err := kvstore.Get(storeKey, nil) + if err != nil { + errCh <- err + return + } + if kvpair == nil { + errCh <- fmt.Errorf("could not fetch Key/Value pair for key %v", storeKey) + return + } + dataCh <- extractStoredData(kvpair, errCh) + close(dataCh) + close(errCh) +} + +func (b KVBackend) getStoredData(watch bool) (<-chan *StoredData, <-chan error) { dataCh := make(chan *StoredData) - errors := make(chan error) + errCh := make(chan error) backend, err := register(b.Name) if err != nil { - errors <- err + go func() { + errCh <- err + }() + return dataCh, errCh } - kvstore, err := valkeyrie.NewStore( backend, b.Client, b.Config, ) + if err != nil { - errors <- err + go func() { + errCh <- err + }() + return dataCh, errCh } - go func() { - stopCh := make(<-chan struct{}) - events, _ := kvstore.Watch(storeKey, stopCh, nil) - for { - kvpair := <-events - storedData, err := getStoredDataFromGzip(kvpair.Value) - if err != nil { - errors <- err - } - dataCh <- storedData - if !watch { - close(dataCh) - close(errors) - } - } - }() + if !watch { + go getSingleData(kvstore, dataCh, errCh) + return dataCh, errCh + } - return dataCh, errors + go loopKV(watch, kvstore, dataCh, errCh) + + return dataCh, errCh } diff --git a/main.go b/main.go index 63c8aec..0037fb0 100644 --- a/main.go +++ b/main.go @@ -1,9 +1,15 @@ package main import ( + "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" + "log" "os" "strconv" + "strings" + "time" "github.com/abronan/valkeyrie/store" "github.com/spf13/cobra" @@ -13,7 +19,7 @@ func main() { var rootCmd = &cobra.Command{ Use: "traefik-certs-dumper", Short: "Dump Let's Encrypt certificates from Traefik", - Long: `Dump the content of the "acme.json" file from Traefik to certificates.`, + Long: `Dump ACME data from Traefik of different storage backends to certificates.`, Version: version, } @@ -22,15 +28,18 @@ func main() { var dumpCmd = &cobra.Command{ Use: "dump", Short: "Dump Let's Encrypt certificates from Traefik", - Long: `Dump the content of the "acme.json" file from Traefik to certificates.`, + Long: `Dump ACME data from Traefik of different storage backends to certificates.`, PreRunE: func(cmd *cobra.Command, args []string) error { source := cmd.Flag("source").Value.String() - sourceFile := cmd.Flag("file").Value.String() - if source == "file" { + sourceFile := cmd.Flag("source.file").Value.String() + watch, _ := strconv.ParseBool(cmd.Flag("watch").Value.String()) + if source == FILE { if _, err := os.Stat(sourceFile); os.IsNotExist(err) { - return fmt.Errorf("--file (%q) does not exist", sourceFile) + return fmt.Errorf("--source.file (%q) does not exist", sourceFile) } - } else if source != "consul" && source != "etcd" && source != "zookeeper" && source != "boltdb" { + } else if source == BOLTDB && watch { + return fmt.Errorf("--watch=true is not supported for boltdb") + } else if source != CONSUL && source != ETCD && source != ZOOKEEPER && source != BOLTDB { return fmt.Errorf("--source (%q) is not allowed, use one of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'", source) } @@ -49,7 +58,57 @@ func main() { RunE: func(cmd *cobra.Command, _ []string) error { source := cmd.Flag("source").Value.String() - acmeFile := cmd.Flag("file").Value.String() + acmeFile := cmd.Flag("source.file").Value.String() + + endpoints := strings.Split(cmd.Flag("source.kv.endpoints").Value.String(), ",") + + storeConfig := &store.Config{} + + timeout, _ := strconv.Atoi(cmd.Flag("source.kv.connection-timeout").Value.String()) + storeConfig.ConnectionTimeout = time.Second * time.Duration(timeout) + storeConfig.Username = cmd.Flag("source.kv.username").Value.String() + storeConfig.Password = cmd.Flag("source.kv.password").Value.String() + + enableTLS, err := strconv.ParseBool(cmd.Flag("source.kv.tls.enable").Value.String()) + if err != nil { + return err + } + + if enableTLS { + tlsConfig := &tls.Config{} + insecureSkipVerify, err := strconv.ParseBool(cmd.Flag("source.kv.tls.insecureskipverify").Value.String()) + if err != nil { + return err + } + tlsConfig.InsecureSkipVerify = insecureSkipVerify + if cmd.Flag("source.kv.tls.ca-cert-file").Value.String() != "" { + caFile := cmd.Flag("source.kv.tls.ca-cert-file").Value.String() + caCert, err := ioutil.ReadFile(caFile) + if err != nil { + log.Fatal(err) + } + roots := x509.NewCertPool() + ok := roots.AppendCertsFromPEM(caCert) + if !ok { + log.Fatalf("failed to parse root certificate") + } + tlsConfig.RootCAs = roots + } + storeConfig.TLS = tlsConfig + } + + // Special parameters for etcd + timeout, _ = strconv.Atoi(cmd.Flag("source.kv.etcd.sync-period").Value.String()) + storeConfig.SyncPeriod = time.Second * time.Duration(timeout) + // Special parameters for boltdb + persistConnection, err := strconv.ParseBool(cmd.Flag("source.kv.boltdb.persist-connection").Value.String()) + if err != nil { + return err + } + storeConfig.PersistConnection = persistConnection + storeConfig.Bucket = cmd.Flag("source.kv.boltdb.bucket").Value.String() + // Special parameters for consul + storeConfig.Token = cmd.Flag("source.kv.consul.token").Value.String() switch source { case "file": @@ -58,28 +117,18 @@ func main() { Path: acmeFile, } case "consul": - config.BackendConfig = KVBackend{ - Name: CONSUL, - Client: []string{"localhost:8500"}, - Config: &store.Config{}, - } + fallthrough case "etcd": - config.BackendConfig = KVBackend{ - Name: ETCD, - Client: []string{"localhost:8500"}, - Config: &store.Config{}, - } + fallthrough case "zookeeper": - config.BackendConfig = KVBackend{ - Name: ZK, - Client: []string{"localhost:8500"}, - Config: &store.Config{}, - } + fallthrough case "boltdb": + fallthrough + default: config.BackendConfig = KVBackend{ - Name: BOLTDB, - Client: []string{"localhost:8500"}, - Config: &store.Config{}, + Name: source, + Client: endpoints, + Config: storeConfig, } } @@ -106,24 +155,26 @@ func main() { }, } - dumpCmd.Flags().String("source", "file", "Source type. One of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'.") - dumpCmd.Flags().String("file", "./acme.json", "Path to 'acme.json' file if source type is 'file'") + dumpCmd.Flags().String("source", "file", "Source type, one of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'. Options for each source type are prefixed with `source..`") + dumpCmd.Flags().String("source.file", "./acme.json", "Path to 'acme.json' for file source.") - /* TODO implement this - dumpCmd.Flags().String("kv.client") - dumpCmd.Flags().String("kv.connection-timeout") - dumpCmd.Flags().String("kv.sync-period") - dumpCmd.Flags().String("kv.bucket") - dumpCmd.Flags().Bool("kv.persist-connection") - dumpCmd.Flags().String("kv.username") - dumpCmd.Flags().String("kv.password") - dumpCmd.Flags().String("kv.token") - dumpCmd.Flags().String("kv.tls-cert-file") - dumpCmd.Flags().String("kv.tls-key-file") - dumpCmd.Flags().String("kv.tls-ca-cert-file") - */ + // Generic parameters for Key/Value backends + dumpCmd.Flags().String("source.kv.endpoints", "localhost:8500", "Comma seperated list of endpoints.") + dumpCmd.Flags().Int("source.kv.connection-timeout", 0, "Connection timeout in seconds.") + dumpCmd.Flags().String("source.kv.password", "", "Password for connection.") + dumpCmd.Flags().String("source.kv.username", "", "Username for connection.") + dumpCmd.Flags().Bool("source.kv.tls.enable", false, "Enable TLS encryption.") + dumpCmd.Flags().Bool("source.kv.tls.insecureskipverify", false, "Trust unverified certificates if TLS is enabled.") + dumpCmd.Flags().String("source.kv.tls.ca-cert-file", "", "Root CA file for certificate verification if TLS is enabled.") + // Special parameters for etcd + dumpCmd.Flags().Int("source.kv.etcd.sync-period", 0, "Sync period for etcd in seconds.") + // Special parameters for boltdb + dumpCmd.Flags().Bool("source.kv.boltdb.persist-connection", false, "Persist connection for boltdb.") + dumpCmd.Flags().String("source.kv.boltdb.bucket", "traefik", "Bucket for boltdb.") + // Special parameters for consul + dumpCmd.Flags().String("source.kv.consul.token", "", "Token for consul.") - dumpCmd.Flags().Bool("watch", true, "Enable watching changes.") + dumpCmd.Flags().Bool("watch", false, "Enable watching changes.") dumpCmd.Flags().String("dest", "./dump", "Path to store the dump content.") dumpCmd.Flags().String("crt-ext", ".crt", "The file extension of the generated certificates.") dumpCmd.Flags().String("crt-name", "certificate", "The file name (without extension) of the generated certificates.") diff --git a/readme.md b/readme.md index 6a33fa4..b3c92c2 100644 --- a/readme.md +++ b/readme.md @@ -44,7 +44,7 @@ docker run ldez/traefik-certs-dumper: ## Usage ```yaml -Dump the content of the "acme.json" file from Traefik to certificates. +Dump ACME data from Traefik of different storage backends to certificates. Usage: traefik-certs-dumper [command] @@ -55,27 +55,40 @@ Available Commands: version Display version Flags: - -h, --help help for certs-dumper - --version version for certs-dumper + -h, --help help for traefik-certs-dumper + --version version for traefik-certs-dumper Use "traefik-certs-dumper [command] --help" for more information about a command. ``` ```yaml -Dump the content of the "acme.json" file from Traefik to certificates. +Dump ACME data from Traefik of different storage backends to certificates. Usage: traefik-certs-dumper dump [flags] Flags: - --crt-ext string The file extension of the generated certificates. (default ".crt") - --crt-name string The file name (without extension) of the generated certificates. (default "certificate") - --dest string Path to store the dump content. (default "./dump") - --domain-subdir Use domain as sub-directory. - -h, --help help for dump - --key-ext string The file extension of the generated private keys. (default ".key") - --key-name string The file name (without extension) of the generated private keys. (default "privatekey") - --source string Path to 'acme.json' file. (default "./acme.json") + --crt-ext string The file extension of the generated certificates. (default ".crt") + --crt-name string The file name (without extension) of the generated certificates. (default "certificate") + --dest string Path to store the dump content. (default "./dump") + --domain-subdir Use domain as sub-directory. + -h, --help help for dump + --key-ext string The file extension of the generated private keys. (default ".key") + --key-name string The file name (without extension) of the generated private keys. (default "privatekey") + --source source.. Source type, one of 'file', 'consul', 'etcd', 'zookeeper', 'boltdb'. Options for each source type are prefixed with source.. (default "file") + --source.file string Path to 'acme.json' for file source. (default "./acme.json") + --source.kv.boltdb.bucket string Bucket for boltdb. (default "traefik") + --source.kv.boltdb.persist-connection Persist connection for boltdb. + --source.kv.connection-timeout int Connection timeout in seconds. + --source.kv.consul.token string Token for consul. + --source.kv.endpoints string Comma seperated list of endpoints. (default "localhost:8500") + --source.kv.etcd.sync-period int Sync period for etcd in seconds. + --source.kv.password string Password for connection. + --source.kv.tls.ca-cert-file string Root CA file for certificate verification if TLS is enabled. + --source.kv.tls.enable Enable TLS encryption. + --source.kv.tls.insecureskipverify Trust unverified certificates if TLS is enabled. + --source.kv.username string Username for connection. + --watch Enable watching changes. ``` ## Examples @@ -93,6 +106,51 @@ dump ``` +### Enabled watching + +```console +$ traefik-certs-dumper dump --watch +2019/04/19 16:56:34 wrote new configuration +dump +├──certs +│ └──my.domain.com.key +└──private + ├──my.domain.com.crt + └──letsencrypt.key +2019/04/19 16:57:14 wrote new configuration +dump +├──certs +│ └──my.domain.com.key +└──private + ├──my.domain.com.crt + └──letsencrypt.key + +``` + +### Consul backend + +```console +$ traefik-certs-dumper dump --source consul --source.kv.endpoints=localhost:8500 +``` + +### Etcd backend + +```console +$ traefik-certs-dumper dump --source etcd --source.kv.endpoints=localhost:2379 +``` + +### Boltdb backend + +```console +$ traefik-certs-dumper dump --source boltdb --source.kv.endpoints=/tmp/my.db +``` + +### Zookeeper backend + +```console +$ traefik-certs-dumper dump --source zookeeper --source.kv.endpoints=localhost:2181 +``` + ### Change source and destination ```console