From 27e0ab455c8e9917b7b599e2177045d332fff56e Mon Sep 17 00:00:00 2001 From: Vladislav Kuzmin Date: Mon, 14 Sep 2020 16:11:57 +0400 Subject: [PATCH] Add isogen executor Relates-To: #344 Change-Id: I1b9b04f1f723d8df6d6aad63b64a917566b22176 --- .../ephemeral/image_configuration.yaml | 2 +- pkg/bootstrap/isogen/command.go | 22 ++- pkg/bootstrap/isogen/command_test.go | 2 +- pkg/bootstrap/isogen/executor.go | 143 ++++++++++++++ pkg/bootstrap/isogen/executor_test.go | 177 ++++++++++++++++++ pkg/events/events.go | 22 ++- pkg/phase/client.go | 4 + 7 files changed, 359 insertions(+), 13 deletions(-) create mode 100644 pkg/bootstrap/isogen/executor.go create mode 100644 pkg/bootstrap/isogen/executor_test.go diff --git a/manifests/function/ephemeral/image_configuration.yaml b/manifests/function/ephemeral/image_configuration.yaml index 06d27667a..6a7d73dd8 100644 --- a/manifests/function/ephemeral/image_configuration.yaml +++ b/manifests/function/ephemeral/image_configuration.yaml @@ -1,7 +1,7 @@ apiVersion: airshipit.org/v1alpha1 kind: ImageConfiguration metadata: - name: default + name: isogen labels: airshipit.org/deploy-k8s: "false" builder: diff --git a/pkg/bootstrap/isogen/command.go b/pkg/bootstrap/isogen/command.go index e1f4e3926..cff11fa35 100644 --- a/pkg/bootstrap/isogen/command.go +++ b/pkg/bootstrap/isogen/command.go @@ -21,7 +21,7 @@ import ( "path/filepath" "strings" - api "opendev.org/airship/airshipctl/pkg/api/v1alpha1" + "opendev.org/airship/airshipctl/pkg/api/v1alpha1" "opendev.org/airship/airshipctl/pkg/bootstrap/cloudinit" "opendev.org/airship/airshipctl/pkg/config" "opendev.org/airship/airshipctl/pkg/container" @@ -35,6 +35,8 @@ const ( ) // GenerateBootstrapIso will generate data for cloud init and start ISO builder container +// TODO (vkuzmin): Remove this public function and move another functions +// to the executor module when the phases will be ready func GenerateBootstrapIso(cfgFactory config.Factory) error { ctx := context.Background() @@ -52,8 +54,8 @@ func GenerateBootstrapIso(cfgFactory config.Factory) error { return err } - imageConfiguration := &api.ImageConfiguration{} - selector, err := document.NewSelector().ByObject(imageConfiguration, api.Scheme) + imageConfiguration := &v1alpha1.ImageConfiguration{} + selector, err := document.NewSelector().ByObject(imageConfiguration, v1alpha1.Scheme) if err != nil { return err } @@ -62,7 +64,7 @@ func GenerateBootstrapIso(cfgFactory config.Factory) error { return err } - err = doc.ToAPIObject(imageConfiguration, api.Scheme) + err = doc.ToAPIObject(imageConfiguration, v1alpha1.Scheme) if err != nil { return err } @@ -78,7 +80,7 @@ func GenerateBootstrapIso(cfgFactory config.Factory) error { return err } - err = generateBootstrapIso(docBundle, builder, doc, imageConfiguration, log.DebugEnabled()) + err = createBootstrapIso(docBundle, builder, doc, imageConfiguration, log.DebugEnabled()) if err != nil { return err } @@ -86,7 +88,7 @@ func GenerateBootstrapIso(cfgFactory config.Factory) error { return verifyArtifacts(imageConfiguration) } -func verifyInputs(cfg *api.ImageConfiguration) error { +func verifyInputs(cfg *v1alpha1.ImageConfiguration) error { if cfg.Container.Volume == "" { return config.ErrMissingConfig{ What: "Must specify volume bind for ISO builder container", @@ -112,7 +114,7 @@ func verifyInputs(cfg *api.ImageConfiguration) error { } func getContainerCfg( - cfg *api.ImageConfiguration, + cfg *v1alpha1.ImageConfiguration, builderCfgYaml []byte, userData []byte, netConf []byte, @@ -126,18 +128,18 @@ func getContainerCfg( return fls } -func verifyArtifacts(cfg *api.ImageConfiguration) error { +func verifyArtifacts(cfg *v1alpha1.ImageConfiguration) error { hostVol := strings.Split(cfg.Container.Volume, ":")[0] metadataPath := filepath.Join(hostVol, cfg.Builder.OutputMetadataFileName) _, err := os.Stat(metadataPath) return err } -func generateBootstrapIso( +func createBootstrapIso( docBundle document.Bundle, builder container.Container, doc document.Document, - cfg *api.ImageConfiguration, + cfg *v1alpha1.ImageConfiguration, debug bool, ) error { cntVol := strings.Split(cfg.Container.Volume, ":")[1] diff --git a/pkg/bootstrap/isogen/command_test.go b/pkg/bootstrap/isogen/command_test.go index a5303ba7d..9bb8acffc 100644 --- a/pkg/bootstrap/isogen/command_test.go +++ b/pkg/bootstrap/isogen/command_test.go @@ -148,7 +148,7 @@ func TestBootstrapIso(t *testing.T) { for _, tt := range tests { outBuf := &bytes.Buffer{} log.Init(tt.debug, outBuf) - actualErr := generateBootstrapIso(bundle, tt.builder, tt.doc, tt.cfg, tt.debug) + actualErr := createBootstrapIso(bundle, tt.builder, tt.doc, tt.cfg, tt.debug) actualOut := outBuf.String() for _, line := range tt.expectedOut { diff --git a/pkg/bootstrap/isogen/executor.go b/pkg/bootstrap/isogen/executor.go new file mode 100644 index 000000000..732c5218b --- /dev/null +++ b/pkg/bootstrap/isogen/executor.go @@ -0,0 +1,143 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package isogen + +import ( + "context" + "io" + + "k8s.io/apimachinery/pkg/runtime/schema" + + "opendev.org/airship/airshipctl/pkg/api/v1alpha1" + "opendev.org/airship/airshipctl/pkg/container" + "opendev.org/airship/airshipctl/pkg/document" + "opendev.org/airship/airshipctl/pkg/errors" + "opendev.org/airship/airshipctl/pkg/events" + "opendev.org/airship/airshipctl/pkg/log" + "opendev.org/airship/airshipctl/pkg/phase/ifc" +) + +var _ ifc.Executor = &Executor{} + +// Executor contains resources for isogen executor +type Executor struct { + ExecutorBundle document.Bundle + ExecutorDocument document.Document + + imgConf *v1alpha1.ImageConfiguration + builder container.Container +} + +// RegisterExecutor adds executor to phase executor registry +func RegisterExecutor(registry map[schema.GroupVersionKind]ifc.ExecutorFactory) error { + obj := &v1alpha1.ImageConfiguration{} + gvks, _, err := v1alpha1.Scheme.ObjectKinds(obj) + if err != nil { + return err + } + registry[gvks[0]] = NewExecutor + return nil +} + +// NewExecutor creates instance of phase executor +func NewExecutor(cfg ifc.ExecutorConfig) (ifc.Executor, error) { + apiObj := &v1alpha1.ImageConfiguration{} + err := cfg.ExecutorDocument.ToAPIObject(apiObj, v1alpha1.Scheme) + if err != nil { + return nil, err + } + + return &Executor{ + ExecutorBundle: cfg.ExecutorBundle, + ExecutorDocument: cfg.ExecutorDocument, + imgConf: apiObj, + }, nil +} + +// Run isogen as a phase runner +func (c *Executor) Run(evtCh chan events.Event, opts ifc.RunOptions) { + defer close(evtCh) + + evtCh <- events.Event{ + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenStart, + }, + } + + if opts.DryRun { + log.Print("command isogen will be executed") + evtCh <- events.Event{ + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenEnd, + }, + } + return + } + + if c.builder == nil { + ctx := context.Background() + builder, err := container.NewContainer( + &ctx, + c.imgConf.Container.ContainerRuntime, + c.imgConf.Container.Image) + c.builder = builder + if err != nil { + handleError(evtCh, err) + return + } + } + + err := createBootstrapIso(c.ExecutorBundle, c.builder, c.ExecutorDocument, c.imgConf, log.DebugEnabled()) + if err != nil { + handleError(evtCh, err) + return + } + + evtCh <- events.Event{ + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenValidation, + }, + } + err = verifyArtifacts(c.imgConf) + if err != nil { + handleError(evtCh, err) + return + } + + evtCh <- events.Event{ + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenEnd, + }, + } +} + +// Validate executor configuration and documents +func (c *Executor) Validate() error { + return errors.ErrNotImplemented{} +} + +// Render executor documents +func (c *Executor) Render(_ io.Writer, _ ifc.RenderOptions) error { + return errors.ErrNotImplemented{} +} + +func handleError(ch chan<- events.Event, err error) { + ch <- events.Event{ + Type: events.ErrorType, + ErrorEvent: events.ErrorEvent{ + Error: err, + }, + } +} diff --git a/pkg/bootstrap/isogen/executor_test.go b/pkg/bootstrap/isogen/executor_test.go new file mode 100644 index 000000000..0282b1b32 --- /dev/null +++ b/pkg/bootstrap/isogen/executor_test.go @@ -0,0 +1,177 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package isogen + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/runtime/schema" + + "opendev.org/airship/airshipctl/pkg/api/v1alpha1" + "opendev.org/airship/airshipctl/pkg/container" + "opendev.org/airship/airshipctl/pkg/document" + "opendev.org/airship/airshipctl/pkg/events" + "opendev.org/airship/airshipctl/pkg/phase/ifc" + "opendev.org/airship/airshipctl/testutil" +) + +var ( + executorDoc = ` +apiVersion: airshipit.org/v1alpha1 +kind: ImageConfiguration +metadata: + name: isogen + labels: + airshipit.org/deploy-k8s: "false" +builder: + networkConfigFileName: network-config + outputMetadataFileName: output-metadata.yaml + userDataFileName: user-data +container: + containerRuntime: docker + image: quay.io/airshipit/isogen:latest-ubuntu_focal + volume: /srv/iso:/config` + executorBundlePath = "testdata/primary/site/test-site/ephemeral/bootstrap" +) + +func TestRegisterExecutor(t *testing.T) { + registry := make(map[schema.GroupVersionKind]ifc.ExecutorFactory) + expectedGVK := schema.GroupVersionKind{ + Group: "airshipit.org", + Version: "v1alpha1", + Kind: "ImageConfiguration", + } + err := RegisterExecutor(registry) + require.NoError(t, err) + + _, found := registry[expectedGVK] + assert.True(t, found) +} + +func TestNewExecutor(t *testing.T) { + execDoc, err := document.NewDocumentFromBytes([]byte(executorDoc)) + require.NoError(t, err) + bundle, err := document.NewBundleByPath(executorBundlePath) + require.NoError(t, err) + _, err = NewExecutor(ifc.ExecutorConfig{ + ExecutorDocument: execDoc, + ExecutorBundle: bundle}) + require.NoError(t, err) +} + +func TestExecutorRun(t *testing.T) { + bundle, err := document.NewBundleByPath(executorBundlePath) + require.NoError(t, err, "Building Bundle Failed") + + tempVol, cleanup := testutil.TempDir(t, "bootstrap-test") + defer cleanup(t) + + volBind := tempVol + ":/dst" + testCfg := &v1alpha1.ImageConfiguration{ + Container: &v1alpha1.Container{ + Volume: volBind, + ContainerRuntime: "docker", + }, + Builder: &v1alpha1.Builder{ + UserDataFileName: "user-data", + NetworkConfigFileName: "net-conf", + }, + } + testDoc := &MockDocument{ + MockAsYAML: func() ([]byte, error) { return []byte("TESTDOC"), nil }, + } + + testCases := []struct { + name string + builder *mockContainer + expectedEvt []events.Event + }{ + { + name: "Run isogen successefully", + builder: &mockContainer{ + runCommand: func() error { return nil }, + getID: func() string { return "TESTID" }, + rmContainer: func() error { return nil }, + }, + expectedEvt: []events.Event{ + { + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenStart, + }, + }, + { + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenValidation, + }, + }, + { + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenEnd, + }, + }, + }, + }, + { + name: "Fail on container command", + builder: &mockContainer{ + runCommand: func() error { + return container.ErrRunContainerCommand{Cmd: "super fail"} + }, + getID: func() string { return "TESTID" }, + rmContainer: func() error { return nil }, + }, + + expectedEvt: []events.Event{ + { + IsogenEvent: events.IsogenEvent{ + Operation: events.IsogenStart, + }, + }, + wrapError(container.ErrRunContainerCommand{Cmd: "super fail"}), + }, + }, + } + for _, test := range testCases { + tt := test + t.Run(tt.name, func(t *testing.T) { + executor := &Executor{ + ExecutorDocument: testDoc, + ExecutorBundle: bundle, + imgConf: testCfg, + builder: tt.builder, + } + require.NoError(t, err) + ch := make(chan events.Event) + go executor.Run(ch, ifc.RunOptions{}) + var actualEvt []events.Event + for evt := range ch { + actualEvt = append(actualEvt, evt) + } + assert.Equal(t, tt.expectedEvt, actualEvt) + }) + } +} + +func wrapError(err error) events.Event { + return events.Event{ + Type: events.ErrorType, + ErrorEvent: events.ErrorEvent{ + Error: err, + }, + } +} diff --git a/pkg/events/events.go b/pkg/events/events.go index 730c8a14b..27dd1108e 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -33,6 +33,8 @@ const ( WaitType // ClusterctlType event emitted by Clusterctl executor ClusterctlType + // IsogenType event emitted by Isogen executor + IsogenType ) // Event holds all possible events that can be produced by airship @@ -42,6 +44,7 @@ type Event struct { ErrorEvent ErrorEvent StatusPollerEvent statuspollerevent.Event ClusterctlEvent ClusterctlEvent + IsogenEvent IsogenEvent } // ErrorEvent is produced when error is encountered @@ -63,7 +66,24 @@ const ( ClusterctlMoveEnd ) -// ClusterctlEvent is prodiced by clusterctl executor +// ClusterctlEvent is produced by clusterctl executor type ClusterctlEvent struct { Operation ClusterctlOperation } + +// IsogenOperation type +type IsogenOperation int + +const ( + // IsogenStart operation + IsogenStart IsogenOperation = iota + // IsogenValidation opearation + IsogenValidation + // IsogenEnd operation + IsogenEnd +) + +// IsogenEvent needs to to track events in isogen executor +type IsogenEvent struct { + Operation IsogenOperation +} diff --git a/pkg/phase/client.go b/pkg/phase/client.go index 0a93c922a..e71681f53 100644 --- a/pkg/phase/client.go +++ b/pkg/phase/client.go @@ -20,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "opendev.org/airship/airshipctl/pkg/api/v1alpha1" + "opendev.org/airship/airshipctl/pkg/bootstrap/isogen" clusterctl "opendev.org/airship/airshipctl/pkg/clusterctl/client" "opendev.org/airship/airshipctl/pkg/document" "opendev.org/airship/airshipctl/pkg/events" @@ -43,6 +44,9 @@ func DefaultExecutorRegistry() map[schema.GroupVersionKind]ifc.ExecutorFactory { if err := applier.RegisterExecutor(execMap); err != nil { log.Fatal(ErrExecutorRegistration{ExecutorName: "kubernetes-apply", Err: err}) } + if err := isogen.RegisterExecutor(execMap); err != nil { + log.Fatal(ErrExecutorRegistration{ExecutorName: "isogen", Err: err}) + } return execMap }