From 6b82a529fcb7fa0eeee377bdc3bd281d57acb074 Mon Sep 17 00:00:00 2001
From: Dmitry Ukov <dukov@mirantis.com>
Date: Sun, 1 Sep 2019 05:14:23 +0000
Subject: [PATCH] [AIR-145] Generate cloud init settings
Settins are generated based on a secret data
Change-Id: Ib4c25e720759694432e03796ae5d1b4f2f2a1a1b
---
pkg/bootstrap/cloudinit/cloud-init.go | 78 +++++++++++++++
pkg/bootstrap/cloudinit/cloud-init_test.go | 67 +++++++++++++
pkg/bootstrap/cloudinit/errors.go | 16 +++
.../cloudinit/testdata/kustomization.yaml | 2 +
pkg/bootstrap/cloudinit/testdata/secret.yaml | 29 ++++++
pkg/bootstrap/isogen/command.go | 86 +++++++++++++++-
pkg/bootstrap/isogen/command_test.go | 97 +++++++++++++++++--
pkg/bootstrap/isogen/config.go | 5 +
.../isogen/testdata/kustomization.yaml | 2 +
pkg/bootstrap/isogen/testdata/secret.yaml | 11 +++
pkg/document/errors.go | 15 +++
pkg/errors/common.go | 8 ++
pkg/util/writefiles.go | 17 ++++
13 files changed, 422 insertions(+), 11 deletions(-)
create mode 100644 pkg/bootstrap/cloudinit/cloud-init.go
create mode 100644 pkg/bootstrap/cloudinit/cloud-init_test.go
create mode 100644 pkg/bootstrap/cloudinit/errors.go
create mode 100644 pkg/bootstrap/cloudinit/testdata/kustomization.yaml
create mode 100644 pkg/bootstrap/cloudinit/testdata/secret.yaml
create mode 100644 pkg/bootstrap/isogen/testdata/kustomization.yaml
create mode 100644 pkg/bootstrap/isogen/testdata/secret.yaml
create mode 100644 pkg/document/errors.go
create mode 100644 pkg/util/writefiles.go
diff --git a/pkg/bootstrap/cloudinit/cloud-init.go b/pkg/bootstrap/cloudinit/cloud-init.go
new file mode 100644
index 000000000..e85d20b3b
--- /dev/null
+++ b/pkg/bootstrap/cloudinit/cloud-init.go
@@ -0,0 +1,78 @@
+package cloudinit
+
+import (
+ b64 "encoding/base64"
+
+ "opendev.org/airship/airshipctl/pkg/document"
+)
+
+const (
+ // TODO (dukov) This should depend on cluster api version once it is
+ // fully available for Metal3. In other words:
+ // - Sectet for v1alpha1
+ // - KubeAdmConfig for v1alpha2
+ EphemeralClusterConfKind = "Secret"
+)
+
+func decodeData(cfg document.Document, key string) ([]byte, error) {
+ data, err := cfg.GetStringMap("data")
+ if err != nil {
+ return nil, ErrDataNotSupplied{DocName: cfg.GetName(), Key: key}
+ }
+
+ res, ok := data[key]
+ if !ok {
+ return nil, ErrDataNotSupplied{DocName: cfg.GetName(), Key: key}
+ }
+
+ return b64.StdEncoding.DecodeString(res)
+}
+
+// getDataFromSecret extracts data from Secret with respect to overrides
+func getDataFromSecret(cfg document.Document, key string) ([]byte, error) {
+ data, err := cfg.GetStringMap("stringData")
+ if err != nil {
+ return decodeData(cfg, key)
+ }
+
+ res, ok := data[key]
+ if !ok {
+ return decodeData(cfg, key)
+ }
+ return []byte(res), nil
+}
+
+// GetCloudData reads YAML document input and generates cloud-init data for
+// node (i.e. Cluster API Machine) with bootstrap annotation.
+func GetCloudData(docBundle document.Bundle, bsAnnotation string) ([]byte, []byte, error) {
+ var userData []byte
+ var netConf []byte
+ docs, err := docBundle.GetByAnnotation(bsAnnotation)
+ if err != nil {
+ return nil, nil, err
+ }
+ var ephemeralCfg document.Document
+ for _, doc := range docs {
+ if doc.GetKind() == EphemeralClusterConfKind {
+ ephemeralCfg = doc
+ break
+ }
+ }
+ if ephemeralCfg == nil {
+ return nil, nil, document.ErrDocNotFound{
+ Annotation: bsAnnotation,
+ Kind: EphemeralClusterConfKind,
+ }
+ }
+
+ netConf, err = getDataFromSecret(ephemeralCfg, "netconfig")
+ if err != nil {
+ return nil, nil, err
+ }
+
+ userData, err = getDataFromSecret(ephemeralCfg, "userdata")
+ if err != nil {
+ return nil, nil, err
+ }
+ return userData, netConf, nil
+}
diff --git a/pkg/bootstrap/cloudinit/cloud-init_test.go b/pkg/bootstrap/cloudinit/cloud-init_test.go
new file mode 100644
index 000000000..e7b76634f
--- /dev/null
+++ b/pkg/bootstrap/cloudinit/cloud-init_test.go
@@ -0,0 +1,67 @@
+package cloudinit
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "opendev.org/airship/airshipctl/pkg/document"
+ "opendev.org/airship/airshipctl/testutil"
+)
+
+func TestGetCloudData(t *testing.T) {
+
+ fSys := testutil.SetupTestFs(t, "testdata")
+ bundle, err := document.NewBundle(fSys, "/", "/")
+ require.NoError(t, err, "Building Bundle Failed")
+
+ tests := []struct {
+ ann string
+ expectedUserData []byte
+ expectedNetData []byte
+ expectedErr error
+ }{
+ {
+ ann: "test=test",
+ expectedUserData: nil,
+ expectedNetData: nil,
+ expectedErr: document.ErrDocNotFound{
+ Annotation: "test=test",
+ Kind: "Secret",
+ },
+ },
+ {
+ ann: "airshipit.org/clustertype=nodata",
+ expectedUserData: nil,
+ expectedNetData: nil,
+ expectedErr: ErrDataNotSupplied{
+ DocName: "node1-bmc-secret1",
+ Key: "netconfig",
+ },
+ },
+ {
+ ann: "test=nodataforcfg",
+ expectedUserData: nil,
+ expectedNetData: nil,
+ expectedErr: ErrDataNotSupplied{
+ DocName: "node1-bmc-secret2",
+ Key: "netconfig",
+ },
+ },
+ {
+ ann: "airshipit.org/clustertype=ephemeral",
+ expectedUserData: []byte("cloud-init"),
+ expectedNetData: []byte("netconfig\n"),
+ expectedErr: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ actualUserData, actualNetData, actualErr := GetCloudData(bundle, tt.ann)
+
+ assert.Equal(t, tt.expectedUserData, actualUserData)
+ assert.Equal(t, tt.expectedNetData, actualNetData)
+ assert.Equal(t, tt.expectedErr, actualErr)
+ }
+}
diff --git a/pkg/bootstrap/cloudinit/errors.go b/pkg/bootstrap/cloudinit/errors.go
new file mode 100644
index 000000000..c076b2773
--- /dev/null
+++ b/pkg/bootstrap/cloudinit/errors.go
@@ -0,0 +1,16 @@
+package cloudinit
+
+import (
+ "fmt"
+)
+
+// ErrDataNotSupplied error returned of no user-data or network configuration
+// in the Secret
+type ErrDataNotSupplied struct {
+ DocName string
+ Key string
+}
+
+func (e ErrDataNotSupplied) Error() string {
+ return fmt.Sprintf("Document %s has no key %s", e.DocName, e.Key)
+}
diff --git a/pkg/bootstrap/cloudinit/testdata/kustomization.yaml b/pkg/bootstrap/cloudinit/testdata/kustomization.yaml
new file mode 100644
index 000000000..97a9721bd
--- /dev/null
+++ b/pkg/bootstrap/cloudinit/testdata/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+ - secret.yaml
diff --git a/pkg/bootstrap/cloudinit/testdata/secret.yaml b/pkg/bootstrap/cloudinit/testdata/secret.yaml
new file mode 100644
index 000000000..d0563b0e4
--- /dev/null
+++ b/pkg/bootstrap/cloudinit/testdata/secret.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ annotations:
+ airshipit.org/clustertype: ephemeral
+ name: node1-bmc-secret
+type: Opaque
+data:
+ netconfig: bmV0Y29uZmlnCg==
+stringData:
+ userdata: cloud-init
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ annotations:
+ airshipit.org/clustertype: nodata
+ name: node1-bmc-secret1
+type: Opaque
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ annotations:
+ test: nodataforcfg
+ name: node1-bmc-secret2
+type: Opaque
+data:
+ foo: bmV0Y29uZmlnCg==
diff --git a/pkg/bootstrap/isogen/command.go b/pkg/bootstrap/isogen/command.go
index 323c2787b..70e811047 100644
--- a/pkg/bootstrap/isogen/command.go
+++ b/pkg/bootstrap/isogen/command.go
@@ -4,10 +4,20 @@ import (
"context"
"fmt"
"io"
+ "path/filepath"
+ "strings"
+ "opendev.org/airship/airshipctl/pkg/bootstrap/cloudinit"
"opendev.org/airship/airshipctl/pkg/container"
+ "opendev.org/airship/airshipctl/pkg/document"
"opendev.org/airship/airshipctl/pkg/errors"
"opendev.org/airship/airshipctl/pkg/util"
+
+ "sigs.k8s.io/kustomize/v3/pkg/fs"
+)
+
+const (
+ builderConfigFileName = "builder-conf.yaml"
)
// GenerateBootstrapIso will generate data for cloud init and start ISO builder container
@@ -24,6 +34,15 @@ func GenerateBootstrapIso(settings *Settings, args []string, out io.Writer) erro
return err
}
+ if err := verifyInputs(cfg, args, out); err != nil {
+ return err
+ }
+
+ docBundle, err := document.NewBundle(fs.MakeRealFS(), args[0], "")
+ if err != nil {
+ return err
+ }
+
fmt.Fprintln(out, "Creating ISO builder container")
builder, err := container.NewContainer(
&ctx, cfg.Container.ContainerRuntime,
@@ -32,17 +51,78 @@ func GenerateBootstrapIso(settings *Settings, args []string, out io.Writer) erro
return err
}
- return generateBootstrapIso(builder, cfg, out, settings.Debug)
+ return generateBootstrapIso(docBundle, builder, cfg, out, settings.Debug)
}
-func generateBootstrapIso(builder container.Container, cfg *Config, out io.Writer, debug bool) error {
+func verifyInputs(cfg *Config, args []string, out io.Writer) error {
+ if len(args) == 0 {
+ fmt.Fprintln(out, "Specify path to document model. Config param from global settings is not supported")
+ return errors.ErrNotImplemented{}
+ }
+
+ if cfg.Container.Volume == "" {
+ fmt.Fprintln(out, "Specify volume bind for ISO builder container")
+ return errors.ErrWrongConfig{}
+ }
+
+ if (cfg.Builder.UserDataFileName == "") || (cfg.Builder.NetworkConfigFileName == "") {
+ fmt.Fprintln(out, "UserDataFileName or NetworkConfigFileName are not specified in ISO builder config")
+ return errors.ErrWrongConfig{}
+ }
+
+ vols := strings.Split(cfg.Container.Volume, ":")
+ switch {
+ case len(vols) == 1:
+ cfg.Container.Volume = fmt.Sprintf("%s:%s", vols[0], vols[0])
+ case len(vols) > 2:
+ fmt.Fprintln(out, "Bad container volume format. Use hostPath:contPath")
+ return errors.ErrWrongConfig{}
+ }
+ return nil
+}
+
+func getContainerCfg(cfg *Config, userData []byte, netConf []byte) (map[string][]byte, error) {
+ hostVol := strings.Split(cfg.Container.Volume, ":")[0]
+
+ fls := make(map[string][]byte)
+ fls[filepath.Join(hostVol, cfg.Builder.UserDataFileName)] = userData
+ fls[filepath.Join(hostVol, cfg.Builder.NetworkConfigFileName)] = netConf
+ builderData, err := cfg.ToYAML()
+ if err != nil {
+ return nil, err
+ }
+ fls[filepath.Join(hostVol, builderConfigFileName)] = builderData
+ return fls, nil
+}
+
+func generateBootstrapIso(
+ docBubdle document.Bundle,
+ builder container.Container,
+ cfg *Config,
+ out io.Writer,
+ debug bool,
+) error {
+ cntVol := strings.Split(cfg.Container.Volume, ":")[1]
+ fmt.Fprintln(out, "Creating cloud-init for ephemeral K8s")
+ userData, netConf, err := cloudinit.GetCloudData(docBubdle, EphemeralClusterAnnotation)
+ if err != nil {
+ return err
+ }
+
+ var fls map[string][]byte
+ fls, err = getContainerCfg(cfg, userData, netConf)
+ if err = util.WriteFiles(fls, 0600); err != nil {
+ return err
+ }
+
vols := []string{cfg.Container.Volume}
+ builderCfgLocation := filepath.Join(cntVol, builderConfigFileName)
fmt.Fprintf(out, "Running default container command. Mounted dir: %s\n", vols)
if err := builder.RunCommand(
[]string{},
nil,
vols,
- []string{},
+ []string{fmt.Sprintf("BUILDER_CONFIG=%s", builderCfgLocation)},
debug,
); err != nil {
return err
diff --git a/pkg/bootstrap/isogen/command_test.go b/pkg/bootstrap/isogen/command_test.go
index df763358e..a814b6bea 100644
--- a/pkg/bootstrap/isogen/command_test.go
+++ b/pkg/bootstrap/isogen/command_test.go
@@ -7,6 +7,11 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "opendev.org/airship/airshipctl/pkg/document"
+ "opendev.org/airship/airshipctl/pkg/errors"
+ "opendev.org/airship/airshipctl/testutil"
)
type mockContainer struct {
@@ -38,9 +43,25 @@ func (mc *mockContainer) GetId() string {
}
func TestBootstrapIso(t *testing.T) {
+ fSys := testutil.SetupTestFs(t, "testdata")
+ bundle, err := document.NewBundle(fSys, "/", "/")
+ require.NoError(t, err, "Building Bundle Failed")
+
+ volBind := "/tmp:/dst"
testErr := fmt.Errorf("TestErr")
+ testCfg := &Config{
+ Container: Container{
+ Volume: volBind,
+ ContainerRuntime: "docker",
+ },
+ Builder: Builder{
+ UserDataFileName: "user-data",
+ NetworkConfigFileName: "net-conf",
+ },
+ }
expOut := []string{
- "Running default container command. Mounted dir: []\n",
+ "Creating cloud-init for ephemeral K8s\n",
+ fmt.Sprintf("Running default container command. Mounted dir: [%s]\n", volBind),
"ISO successfully built.\n",
"Debug flag is set. Container TESTID stopped but not deleted.\n",
"Removing container.\n",
@@ -57,9 +78,9 @@ func TestBootstrapIso(t *testing.T) {
builder: &mockContainer{
runCommand: func() error { return testErr },
},
- cfg: &Config{},
+ cfg: testCfg,
debug: false,
- expectedOut: expOut[0],
+ expectedOut: expOut[0] + expOut[1],
expectdErr: testErr,
},
{
@@ -67,9 +88,9 @@ func TestBootstrapIso(t *testing.T) {
runCommand: func() error { return nil },
getId: func() string { return "TESTID" },
},
- cfg: &Config{},
+ cfg: testCfg,
debug: true,
- expectedOut: expOut[0] + expOut[1] + expOut[2],
+ expectedOut: expOut[0] + expOut[1] + expOut[2] + expOut[3],
expectdErr: nil,
},
{
@@ -78,16 +99,16 @@ func TestBootstrapIso(t *testing.T) {
getId: func() string { return "TESTID" },
rmContainer: func() error { return testErr },
},
- cfg: &Config{},
+ cfg: testCfg,
debug: false,
- expectedOut: expOut[0] + expOut[1] + expOut[3],
+ expectedOut: expOut[0] + expOut[1] + expOut[2] + expOut[4],
expectdErr: testErr,
},
}
for _, tt := range tests {
actualOut := bytes.NewBufferString("")
- actualErr := generateBootstrapIso(tt.builder, tt.cfg, actualOut, tt.debug)
+ actualErr := generateBootstrapIso(bundle, tt.builder, tt.cfg, actualOut, tt.debug)
errS := fmt.Sprintf("generateBootstrapIso should have return error %s, got %s", tt.expectdErr, actualErr)
assert.Equal(t, actualErr, tt.expectdErr, errS)
@@ -96,3 +117,63 @@ func TestBootstrapIso(t *testing.T) {
assert.Equal(t, actualOut.String(), tt.expectedOut, errS)
}
}
+
+func TestVerifyInputs(t *testing.T) {
+ tests := []struct {
+ cfg *Config
+ args []string
+ expectedErr error
+ }{
+ {
+ cfg: &Config{},
+ args: []string{},
+ expectedErr: errors.ErrNotImplemented{},
+ },
+ {
+ cfg: &Config{},
+ args: []string{"."},
+ expectedErr: errors.ErrWrongConfig{},
+ },
+ {
+ cfg: &Config{
+ Container: Container{
+ Volume: "/tmp:/dst",
+ },
+ },
+ args: []string{"."},
+ expectedErr: errors.ErrWrongConfig{},
+ },
+ {
+ cfg: &Config{
+ Container: Container{
+ Volume: "/tmp",
+ },
+ Builder: Builder{
+ UserDataFileName: "user-data",
+ NetworkConfigFileName: "net-conf",
+ },
+ },
+ args: []string{"."},
+ expectedErr: nil,
+ },
+ {
+ cfg: &Config{
+ Container: Container{
+ Volume: "/tmp:/dst:/dst1",
+ },
+ Builder: Builder{
+ UserDataFileName: "user-data",
+ NetworkConfigFileName: "net-conf",
+ },
+ },
+ args: []string{"."},
+ expectedErr: errors.ErrWrongConfig{},
+ },
+ }
+
+ for _, tt := range tests {
+ actualErr := verifyInputs(tt.cfg, tt.args, bytes.NewBufferString(""))
+ assert.Equal(t, tt.expectedErr, actualErr)
+ }
+
+}
diff --git a/pkg/bootstrap/isogen/config.go b/pkg/bootstrap/isogen/config.go
index 50bb7fe48..800548144 100644
--- a/pkg/bootstrap/isogen/config.go
+++ b/pkg/bootstrap/isogen/config.go
@@ -8,6 +8,11 @@ import (
"opendev.org/airship/airshipctl/pkg/environment"
)
+const (
+ // TODO this should be part of a airshipctl config
+ EphemeralClusterAnnotation = "airshipit.org/clustertype=ephemeral"
+)
+
// Settings settings for isogen command
type Settings struct {
*environment.AirshipCTLSettings
diff --git a/pkg/bootstrap/isogen/testdata/kustomization.yaml b/pkg/bootstrap/isogen/testdata/kustomization.yaml
new file mode 100644
index 000000000..97a9721bd
--- /dev/null
+++ b/pkg/bootstrap/isogen/testdata/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+ - secret.yaml
diff --git a/pkg/bootstrap/isogen/testdata/secret.yaml b/pkg/bootstrap/isogen/testdata/secret.yaml
new file mode 100644
index 000000000..ebc6976c3
--- /dev/null
+++ b/pkg/bootstrap/isogen/testdata/secret.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ annotations:
+ airshipit.org/clustertype: ephemeral
+ name: node1-bmc-secret
+type: Opaque
+data:
+ netconfig: bmV0Y29uZmlnCg==
+stringData:
+ userdata: cloud-init
diff --git a/pkg/document/errors.go b/pkg/document/errors.go
new file mode 100644
index 000000000..38a4c95fa
--- /dev/null
+++ b/pkg/document/errors.go
@@ -0,0 +1,15 @@
+package document
+
+import (
+ "fmt"
+)
+
+// ErrDocNotFound returned if desired document not found
+type ErrDocNotFound struct {
+ Annotation string
+ Kind string
+}
+
+func (e ErrDocNotFound) Error() string {
+ return fmt.Sprintf("Document annotated by %s with Kind %s not found", e.Annotation, e.Kind)
+}
diff --git a/pkg/errors/common.go b/pkg/errors/common.go
index 085be7b1d..16ec47ce1 100644
--- a/pkg/errors/common.go
+++ b/pkg/errors/common.go
@@ -7,3 +7,11 @@ type ErrNotImplemented struct {
func (e ErrNotImplemented) Error() string {
return "Error. Not implemented"
}
+
+// ErrWrongConfig returned in case of incorrect configuration
+type ErrWrongConfig struct {
+}
+
+func (e ErrWrongConfig) Error() string {
+ return "Error. Wrong configuration"
+}
diff --git a/pkg/util/writefiles.go b/pkg/util/writefiles.go
new file mode 100644
index 000000000..1ec393815
--- /dev/null
+++ b/pkg/util/writefiles.go
@@ -0,0 +1,17 @@
+package util
+
+import (
+ "io/ioutil"
+ "os"
+)
+
+// WriteFiles write multiple files described in a map
+func WriteFiles(fls map[string][]byte, mode os.FileMode) error {
+ for fileName, data := range fls {
+ if err := ioutil.WriteFile(fileName, data, mode); err != nil {
+ return err
+ }
+ }
+ return nil
+
+}