From 732e325df248bef80283e0feb75744b706dd22a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Mon, 15 Apr 2019 00:08:34 +0200 Subject: [PATCH] add consul watcher --- dumper.go | 121 +++++++++++++++++++++++++++++++++++++++++++++++------- go.mod | 5 +++ go.sum | 59 ++++++++++++++++++++++++++ main.go | 72 +++++++++++--------------------- 4 files changed, 194 insertions(+), 63 deletions(-) diff --git a/dumper.go b/dumper.go index be31822..114dcae 100644 --- a/dumper.go +++ b/dumper.go @@ -1,12 +1,17 @@ package main import ( + "bytes" + "compress/gzip" "encoding/json" "encoding/pem" + "fmt" "io/ioutil" "os" "path/filepath" + "github.com/hashicorp/consul/api" + consulwatch "github.com/hashicorp/consul/watch" "github.com/xenolf/lego/certcrypto" "github.com/xenolf/lego/registration" ) @@ -50,33 +55,24 @@ type fileInfo struct { Ext string } -func dump(acmeFile, dumpPath string, crtInfo, keyInfo fileInfo, domainSubDir bool) error { - f, err := os.Open(acmeFile) - if err != nil { - return err - } +func dump(data StoredData, dumpPath string, crtInfo, keyInfo fileInfo, domainSubDir bool) error { - data := StoredData{} - if err = json.NewDecoder(f).Decode(&data); err != nil { - return err - } - - if err = os.RemoveAll(dumpPath); err != nil { + if err := os.RemoveAll(dumpPath); err != nil { return err } if !domainSubDir { - if err = os.MkdirAll(filepath.Join(dumpPath, certsSubDir), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(dumpPath, certsSubDir), 0755); err != nil { return err } } - if err = os.MkdirAll(filepath.Join(dumpPath, keysSubDir), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(dumpPath, keysSubDir), 0755); err != nil { return err } privateKeyPem := extractPEMPrivateKey(data.Account) - err = ioutil.WriteFile(filepath.Join(dumpPath, keysSubDir, "letsencrypt"+keyInfo.Ext), privateKeyPem, 0666) + err := ioutil.WriteFile(filepath.Join(dumpPath, keysSubDir, "letsencrypt"+keyInfo.Ext), privateKeyPem, 0666) if err != nil { return err } @@ -96,6 +92,63 @@ func dump(acmeFile, dumpPath string, crtInfo, keyInfo fileInfo, domainSubDir boo return nil } +func dumpFile(acmeFile string, dumpPath string, crtInfo, keyInfo fileInfo, domainSubDir bool) error { + f, err := os.Open(acmeFile) + if err != nil { + return err + } + + data := StoredData{} + if err = json.NewDecoder(f).Decode(&data); err != nil { + return err + } + + return dump(data, dumpPath, crtInfo, keyInfo, domainSubDir) +} + +func dumpConsul(watch bool, dumpPath string, crtInfo, keyInfo fileInfo, domainSubDir bool) { + + params := map[string]interface{}{ + "type": "key", + "key": "traefik/acme/account/object", + } + plan, _ := consulwatch.Parse(params) + plan.Handler = func(idx uint64, data interface{}) { + + // TODO is here a better way? + var buf bytes.Buffer + json.NewEncoder(&buf).Encode(data) + kvpair := api.KVPair{} + json.Unmarshal(buf.Bytes(), &kvpair) + + r, err := gzip.NewReader(bytes.NewBuffer(kvpair.Value)) + defer r.Close() + if err != nil { + fmt.Printf("[ERR] %s", err) + } + + acmeData, err := ioutil.ReadAll(r) + if err != nil { + fmt.Printf("[ERR] %s", err) + } + + storedData := StoredData{} + json.Unmarshal(acmeData, &storedData) + + dump(storedData, dumpPath, crtInfo, keyInfo, domainSubDir) + if err := tree(dumpPath, ""); err != nil { + fmt.Printf("[ERR] %s", err) + } + if !watch { + plan.Stop() + } + } + + config := api.DefaultConfig() + fmt.Println("Start watching consul...") + plan.Run(config.Address) +} + func writeCert(dumpPath string, cert *Certificate, info fileInfo, domainSubDir bool) error { certPath := filepath.Join(dumpPath, certsSubDir, cert.Domain.Main+info.Ext) if domainSubDir { @@ -139,3 +192,43 @@ func extractPEMPrivateKey(account *Account) []byte { return pem.EncodeToMemory(block) } + +func tree(root, indent string) error { + fi, err := os.Stat(root) + if err != nil { + return fmt.Errorf("could not stat %s: %v", root, err) + } + + fmt.Println(fi.Name()) + if !fi.IsDir() { + return nil + } + + fis, err := ioutil.ReadDir(root) + if err != nil { + return fmt.Errorf("could not read dir %s: %v", root, err) + } + + var names []string + for _, fi := range fis { + if fi.Name()[0] != '.' { + names = append(names, fi.Name()) + } + } + + for i, name := range names { + add := "│ " + if i == len(names)-1 { + fmt.Printf(indent + "└──") + add = " " + } else { + fmt.Printf(indent + "├──") + } + + if err := tree(filepath.Join(root, name), indent+add); err != nil { + return err + } + } + + return nil +} diff --git a/go.mod b/go.mod index 9ef9c76..a9760c1 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,12 @@ module github.com/ldez/traefik-certs-dumper require ( github.com/cenkalti/backoff v2.1.1+incompatible // indirect + github.com/hashicorp/consul v1.4.4 + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect + github.com/hashicorp/go-rootcerts v1.0.0 // indirect + github.com/hashicorp/serf v0.8.3 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect github.com/xenolf/lego v2.2.0+incompatible diff --git a/go.sum b/go.sum index 945f965..0ae3e71 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,80 @@ +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/hashicorp/consul v1.4.4 h1:DR1+5EGgnPsd/LIsK3c9RDvajcsV5GOkGQBSNd3dpn8= +github.com/hashicorp/consul v1.4.4/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= +github.com/hashicorp/consul/api v1.0.1 h1:LkHu3cLXjya4lgrAyZVe/CUBXgJ7AcDWKSeCjAYN9w0= +github.com/hashicorp/consul/api v1.0.1/go.mod h1:LQlewHPiuaRhn1mP2XE4RrjnlRgOeWa/ZM0xWLCen2M= +github.com/hashicorp/consul/sdk v0.1.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.8.3 h1:MWYcmct5EtKz0efYooPcL0yNkem+7kWxqXDi/UIh+8k= +github.com/hashicorp/serf v0.8.3/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/xenolf/lego v2.2.0+incompatible h1:r4UAcpgPmX3j0aThoVrRM1FFLcvyy08UyGbIwFU4zoQ= github.com/xenolf/lego v2.2.0+incompatible/go.mod h1:fwiGnfsIjG7OHPfOvgK7Y/Qo6+2Ox0iozjNTkZICKbY= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f h1:ETU2VEl7TnT5bl7IvuKEzTDpplg5wzGYsOCAPhdoEIg= golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/main.go b/main.go index b629b69..51813af 100644 --- a/main.go +++ b/main.go @@ -2,9 +2,7 @@ package main import ( "fmt" - "io/ioutil" "os" - "path/filepath" "strconv" "github.com/spf13/cobra" @@ -23,6 +21,11 @@ func main() { Short: "Dump Let's Encrypt certificates from Traefik", Long: `Dump the content of the "acme.json" file from Traefik to certificates.`, PreRunE: func(cmd *cobra.Command, args []string) error { + source := cmd.Flag("source").Value.String() + if source != "file" && source != "consul" { + return fmt.Errorf("--source (%q) is not allowed, use one of 'file' or 'consul'", source) + } + crtExt := cmd.Flag("crt-ext").Value.String() keyExt := cmd.Flag("key-ext").Value.String() @@ -36,7 +39,7 @@ func main() { return nil }, RunE: func(cmd *cobra.Command, _ []string) error { - acmeFile := cmd.Flag("source").Value.String() + acmeFile := cmd.Flag("source-file").Value.String() dumpPath := cmd.Flag("dest").Value.String() crtInfo := fileInfo{ @@ -50,17 +53,28 @@ func main() { } subDir, _ := strconv.ParseBool(cmd.Flag("domain-subdir").Value.String()) + watchConsul, _ := strconv.ParseBool(cmd.Flag("source-consul-watch").Value.String()) - err := dump(acmeFile, dumpPath, crtInfo, keyInfo, subDir) - if err != nil { - return err + switch cmd.Flag("source").Value.String() { + + case "consul": + dumpConsul(watchConsul, dumpPath, crtInfo, keyInfo, subDir) + + case "file": + default: + err := dumpFile(acmeFile, dumpPath, crtInfo, keyInfo, subDir) + if err != nil { + return err + } + return tree(dumpPath, "") } - - return tree(dumpPath, "") + return nil }, } - dumpCmd.Flags().String("source", "./acme.json", "Path to 'acme.json' file.") + dumpCmd.Flags().String("source", "file", "Source type. One of 'file' or 'consul'. Consul connection parameters can be set via environment variables, see https://www.consul.io/docs/commands/index.html#environment-variables") + dumpCmd.Flags().String("source-file", "./acme.json", "Path to 'acme.json' file if source type is 'file'") + dumpCmd.Flags().Bool("source-consul-watch", true, "Enable watching changes in Consul.") dumpCmd.Flags().String("dest", "./dump", "Path to store the dump content.") dumpCmd.Flags().String("crt-ext", ".crt", "The file extension of the generated certificates.") dumpCmd.Flags().String("crt-name", "certificate", "The file name (without extension) of the generated certificates.") @@ -84,43 +98,3 @@ func main() { os.Exit(1) } } - -func tree(root, indent string) error { - fi, err := os.Stat(root) - if err != nil { - return fmt.Errorf("could not stat %s: %v", root, err) - } - - fmt.Println(fi.Name()) - if !fi.IsDir() { - return nil - } - - fis, err := ioutil.ReadDir(root) - if err != nil { - return fmt.Errorf("could not read dir %s: %v", root, err) - } - - var names []string - for _, fi := range fis { - if fi.Name()[0] != '.' { - names = append(names, fi.Name()) - } - } - - for i, name := range names { - add := "│ " - if i == len(names)-1 { - fmt.Printf(indent + "└──") - add = " " - } else { - fmt.Printf(indent + "├──") - } - - if err := tree(filepath.Join(root, name), indent+add); err != nil { - return err - } - } - - return nil -}