From f5cf2d379a5e2f46b7d42849d942e501dacae17c Mon Sep 17 00:00:00 2001
From: Kostiantyn Kalynovskyi <kkalynovskyi@mirantis.com>
Date: Tue, 16 Jun 2020 16:33:08 -0500
Subject: [PATCH] Add applier from cli-utils package

The commit adds applier object from cli-utils which would allow us
to wait for kubernetes objects to reach required states and support
pruning.

Integration is very similar to the one with kubectl, because backend
mechanism for reading manifests is the same. We need to write them to
temporary directory in $HOME/.airship directory, and point cli-utils
to it. It will then take care of storing what was applied to k8s cluster
for prunning and also will be capable of waiting for core resources to
reach required state by constantly polling them.

To integrate we need to create a namespace in which we will store
configmap that keeps track of what objects were applied before and enable
prunning

Relates-To: #238

Change-Id: I4c0123cc57b78bd13dbe320e0ab9f28bbed2301d
---
 pkg/k8s/applier/applier.go                    | 259 ++++++++++++++++++
 pkg/k8s/applier/applier_test.go               | 165 +++++++++++
 pkg/k8s/applier/apply_options.go              |  27 ++
 pkg/k8s/applier/errors.go                     |  37 +++
 pkg/k8s/applier/fake.go                       | 128 +++++++++
 pkg/k8s/applier/testdata/kubeconfig.yaml      |  19 ++
 .../testdata/source_bundle/kustomization.yaml |   2 +
 .../source_bundle/replicationcontroller.yaml  |  22 ++
 .../testdata/two_cm_bundle/kustomization.yaml |   2 +
 .../testdata/two_cm_bundle/resources.yaml     |  40 +++
 10 files changed, 701 insertions(+)
 create mode 100644 pkg/k8s/applier/applier.go
 create mode 100644 pkg/k8s/applier/applier_test.go
 create mode 100644 pkg/k8s/applier/apply_options.go
 create mode 100644 pkg/k8s/applier/errors.go
 create mode 100644 pkg/k8s/applier/fake.go
 create mode 100644 pkg/k8s/applier/testdata/kubeconfig.yaml
 create mode 100644 pkg/k8s/applier/testdata/source_bundle/kustomization.yaml
 create mode 100644 pkg/k8s/applier/testdata/source_bundle/replicationcontroller.yaml
 create mode 100644 pkg/k8s/applier/testdata/two_cm_bundle/kustomization.yaml
 create mode 100644 pkg/k8s/applier/testdata/two_cm_bundle/resources.yaml

diff --git a/pkg/k8s/applier/applier.go b/pkg/k8s/applier/applier.go
new file mode 100644
index 000000000..3cb78abf0
--- /dev/null
+++ b/pkg/k8s/applier/applier.go
@@ -0,0 +1,259 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package applier
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	v1 "k8s.io/api/core/v1"
+	apierror "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/cli-runtime/pkg/resource"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	cliapply "sigs.k8s.io/cli-utils/pkg/apply"
+	applyevent "sigs.k8s.io/cli-utils/pkg/apply/event"
+	"sigs.k8s.io/cli-utils/pkg/apply/poller"
+	clicommon "sigs.k8s.io/cli-utils/pkg/common"
+	"sigs.k8s.io/cli-utils/pkg/manifestreader"
+	"sigs.k8s.io/yaml"
+
+	"opendev.org/airship/airshipctl/pkg/document"
+	"opendev.org/airship/airshipctl/pkg/events"
+	"opendev.org/airship/airshipctl/pkg/k8s/utils"
+	"opendev.org/airship/airshipctl/pkg/log"
+)
+
+const (
+	// DefaultNamespace to store inventory objects in
+	DefaultNamespace = "airshipit"
+)
+
+// Applier delivers documents to kubernetes in a declerative way
+type Applier struct {
+	Driver                Driver
+	Factory               cmdutil.Factory
+	Streams               genericclioptions.IOStreams
+	Poller                poller.Poller
+	ManifestReaderFactory utils.ManifestReaderFactory
+}
+
+// ReaderFactory function that returns reader factory interface
+type ReaderFactory func(validate bool, bundle document.Bundle, factory cmdutil.Factory) manifestreader.ManifestReader
+
+// NewApplier returns instance of Applier
+func NewApplier(f cmdutil.Factory, streams genericclioptions.IOStreams) *Applier {
+	return &Applier{
+		Factory:               f,
+		Streams:               streams,
+		ManifestReaderFactory: utils.DefaultManifestReaderFactory,
+		Driver: &Adaptor{
+			СliUtilsApplier: cliapply.NewApplier(f, streams),
+		},
+	}
+}
+
+// ApplyBundle apply bundle to kubernetes cluster
+func (a *Applier) ApplyBundle(bundle document.Bundle, ao ApplyOptions) <-chan events.Event {
+	eventCh := make(chan events.Event)
+	go func() {
+		defer close(eventCh)
+		if bundle == nil {
+			// TODO add this to errors
+			handleError(eventCh, ErrApplyNilBundle{})
+			return
+		}
+		log.Printf("Applying bundle, inventory id: %s", ao.BundleName)
+		// TODO Get this selector from document package instead
+		// Selector to filter invenotry document from bundle
+		selector := document.
+			NewSelector().
+			ByLabel(clicommon.InventoryLabel).
+			ByKind(document.ConfigMapKind)
+		// if we could find exactly one inventory document, we don't do anything else with it
+		_, err := bundle.SelectOne(selector)
+		// if we got an error, which means we could not find Config Map with invetory ID at rest
+		// now we need to generate and inject one at runtime
+		if err != nil && errors.As(err, &document.ErrDocNotFound{}) {
+			log.Debug("Inventory Object config Map not found, auto generating Invetory object")
+			invDoc, innerErr := NewInventoryDocument(ao.BundleName)
+			if innerErr != nil {
+				// this should never happen
+				log.Debug("Failed to create new invetory document")
+				handleError(eventCh, innerErr)
+				return
+			}
+			log.Debugf("Injecting Invetory Object: %v into bundle", invDoc)
+			innerErr = bundle.Append(invDoc)
+			if innerErr != nil {
+				log.Debug("Couldn't append bunlde with inventory document")
+				handleError(eventCh, innerErr)
+				return
+			}
+			log.Debugf("Making sure that inventory object namespace %s exists", invDoc.GetNamespace())
+			innerErr = a.ensureNamespaceExists(invDoc.GetNamespace())
+			if innerErr != nil {
+				handleError(eventCh, innerErr)
+				return
+			}
+		} else if err != nil {
+			handleError(eventCh, err)
+			return
+		}
+		err = a.Driver.Initialize(a.Poller)
+		if err != nil {
+			handleError(eventCh, err)
+			return
+		}
+		ctx := context.Background()
+		infos, err := a.ManifestReaderFactory(false, bundle, a.Factory).Read()
+		if err != nil {
+			handleError(eventCh, err)
+			return
+		}
+		ch := a.Driver.Run(ctx, infos, cliApplyOptions(ao))
+		for e := range ch {
+			eventCh <- events.Event{
+				Type:         events.ApplierType,
+				ApplierEvent: e,
+			}
+		}
+	}()
+	return eventCh
+}
+
+func (a *Applier) ensureNamespaceExists(name string) error {
+	clientSet, err := a.Factory.KubernetesClientSet()
+	if err != nil {
+		return err
+	}
+	nsClient := clientSet.CoreV1().Namespaces()
+	_, err = nsClient.Create(&v1.Namespace{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Namespace",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	})
+	if err != nil && !apierror.IsAlreadyExists(err) {
+		// we got error and error doesn't say that NS already exist
+		return err
+	}
+	return nil
+}
+
+func cliApplyOptions(ao ApplyOptions) cliapply.Options {
+	var emitStatusEvents bool
+	// if wait timeout is 0, we don't want to status poller to emit any events,
+	// this should disable waiting for resources
+	if ao.WaitTimeout != time.Duration(0) {
+		emitStatusEvents = true
+	}
+	return cliapply.Options{
+		EmitStatusEvents: emitStatusEvents,
+		ReconcileTimeout: ao.WaitTimeout,
+		NoPrune:          !ao.Prune,
+		DryRun:           ao.DryRun,
+	}
+}
+
+// Driver to cli-utils apply
+type Driver interface {
+	Initialize(p poller.Poller) error
+	Run(ctx context.Context, infos []*resource.Info, options cliapply.Options) <-chan applyevent.Event
+}
+
+// Adaptor is implementation of driver interface
+type Adaptor struct {
+	СliUtilsApplier *cliapply.Applier
+	Factory         cmdutil.Factory
+}
+
+// Initialize sets fake required command line flags for underlaying cli-utils package
+func (a *Adaptor) Initialize(p poller.Poller) error {
+	cmd := &cobra.Command{}
+	// Code below is copied from cli-utils package and used the same way as in upstream:
+	// https://github.com/kubernetes-sigs/cli-utils/blob/v0.14.0/cmd/apply/cmdapply.go#L35-L46
+	// Skip error checking is done in a same way as in upstream usage of this package
+	err := a.СliUtilsApplier.SetFlags(cmd)
+	if err != nil {
+		return err
+	}
+	var unusedBool bool
+	cmd.Flags().BoolVar(&unusedBool, "dry-run", unusedBool, "NOT USED")
+	cmd.Flags().MarkHidden("dry-run") //nolint:errcheck
+	cmdutil.AddValidateFlags(cmd)
+	cmd.Flags().MarkHidden("validate") //nolint:errcheck
+	cmdutil.AddServerSideApplyFlags(cmd)
+	cmd.Flags().MarkHidden("server-side")     //nolint:errcheck
+	cmd.Flags().MarkHidden("force-conflicts") //nolint:errcheck
+	cmd.Flags().MarkHidden("field-manager")   //nolint:errcheck
+	err = a.СliUtilsApplier.Initialize(cmd)
+	if err != nil {
+		return err
+	}
+	// status poller needs to be set only after the Initialize method is executed
+	if p != nil {
+		a.СliUtilsApplier.StatusPoller = p
+	}
+	return nil
+}
+
+// Run perform apply operation
+func (a *Adaptor) Run(ctx context.Context, infos []*resource.Info, options cliapply.Options) <-chan applyevent.Event {
+	return a.СliUtilsApplier.Run(ctx, infos, options)
+}
+
+// NewInventoryDocument returns default config map with invetory Id to group up the objects
+func NewInventoryDocument(inventoryID string) (document.Document, error) {
+	cm := v1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       document.ConfigMapKind,
+			APIVersion: document.ConfigMapVersion,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			// name here is a dummy name, cli utils won't use this name, this config map simply parsed
+			// for invetory ID from label, and then ignores this config map
+			Name: fmt.Sprintf("%s-%s", "airshipit", inventoryID),
+			// TODO this limits us to single namespace. research passing this as parameter from
+			// somewhere higher level package that uses this module
+			Namespace: DefaultNamespace,
+			Labels: map[string]string{
+				clicommon.InventoryLabel: inventoryID,
+			},
+		},
+		Data: map[string]string{},
+	}
+	b, err := yaml.Marshal(cm)
+	if err != nil {
+		return nil, err
+	}
+	return document.NewDocumentFromBytes(b)
+}
+
+func handleError(ch chan<- events.Event, err error) {
+	ch <- events.Event{
+		Type: events.ErrorType,
+		ErrorEvent: events.ErrorEvent{
+			Error: err,
+		},
+	}
+}
diff --git a/pkg/k8s/applier/applier_test.go b/pkg/k8s/applier/applier_test.go
new file mode 100644
index 000000000..f36df75d3
--- /dev/null
+++ b/pkg/k8s/applier/applier_test.go
@@ -0,0 +1,165 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package applier_test
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"sigs.k8s.io/cli-utils/pkg/apply/event"
+	"sigs.k8s.io/cli-utils/pkg/apply/poller"
+
+	"opendev.org/airship/airshipctl/pkg/document"
+	"opendev.org/airship/airshipctl/pkg/events"
+	"opendev.org/airship/airshipctl/pkg/k8s/applier"
+	"opendev.org/airship/airshipctl/testutil"
+	k8stest "opendev.org/airship/airshipctl/testutil/k8sutils"
+)
+
+func TestFakeApplier(t *testing.T) {
+	a := applier.NewFakeApplier(genericclioptions.IOStreams{
+		In:     os.Stdin,
+		Out:    os.Stdout,
+		ErrOut: os.Stderr,
+	}, k8stest.SuccessEvents(), k8stest.FakeFactory(t, []k8stest.ClientHandler{}))
+	assert.NotNil(t, a)
+}
+
+// TODO Develop more coverage with tests
+func TestApplierRun(t *testing.T) {
+	bundle := testutil.NewTestBundle(t, "testdata/source_bundle")
+	replicationController, err := bundle.SelectOne(document.NewSelector().ByKind("ReplicationController"))
+	require.NoError(t, err)
+	b, err := replicationController.AsYAML()
+	require.NoError(t, err)
+	f := k8stest.FakeFactory(t,
+		[]k8stest.ClientHandler{
+			&k8stest.InventoryObjectHandler{},
+			&k8stest.NamespaceHandler{},
+			&k8stest.GenericHandler{
+				Obj:       &corev1.ReplicationController{},
+				Bytes:     b,
+				URLPath:   "/namespaces/%s/replicationcontrollers",
+				Namespace: replicationController.GetNamespace(),
+			},
+		})
+	defer f.Cleanup()
+	s := genericclioptions.IOStreams{
+		In:     os.Stdin,
+		Out:    os.Stdout,
+		ErrOut: os.Stderr,
+	}
+	tests := []struct {
+		name           string
+		driver         applier.Driver
+		expectErr      bool
+		expectedString string
+		bundle         document.Bundle
+		poller         poller.Poller
+	}{
+		{
+			name:           "init-err",
+			driver:         applier.NewFakeAdaptor().WithInitError(fmt.Errorf("init-err")),
+			expectedString: "init-err",
+			bundle:         bundle,
+			expectErr:      true,
+		},
+		{
+			name:           "can't reach cluster",
+			expectedString: "connection refused",
+			expectErr:      true,
+			bundle:         bundle,
+			poller:         &applier.FakePoller{},
+		},
+		{
+			name:           "bundle failure",
+			expectedString: "Can not apply nil bundle, airship.kubernetes.Client",
+			expectErr:      true,
+		},
+		{
+			name:      "success",
+			expectErr: false,
+			bundle:    bundle,
+			driver:    applier.NewFakeAdaptor().WithEvents(k8stest.SuccessEvents()),
+		},
+		{
+			name:      "set poller",
+			expectErr: false,
+			bundle:    bundle,
+			driver:    applier.NewFakeAdaptor().WithEvents(k8stest.SuccessEvents()),
+			poller:    &applier.FakePoller{},
+		},
+		{
+			name:           "two configmaps present",
+			expectErr:      true,
+			bundle:         newBundle("testdata/two_cm_bundle", t),
+			driver:         applier.NewFakeAdaptor().WithEvents(k8stest.SuccessEvents()),
+			expectedString: "found more than one document",
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			// create default applier
+			a := applier.NewApplier(f, s)
+			opts := applier.ApplyOptions{
+				WaitTimeout: time.Second * 5,
+				BundleName:  "test-bundle",
+				DryRun:      true,
+			}
+			if tt.driver != nil {
+				a.Driver = tt.driver
+			}
+			if tt.poller != nil {
+				a.Poller = tt.poller
+			}
+			ch := a.ApplyBundle(tt.bundle, opts)
+			var airEvents []events.Event
+			for e := range ch {
+				airEvents = append(airEvents, e)
+			}
+			var errs []error
+			for _, e := range airEvents {
+				if e.Type == events.ErrorType {
+					errs = append(errs, e.ErrorEvent.Error)
+				} else if e.Type == events.ApplierType && e.ApplierEvent.Type == event.ErrorType {
+					errs = append(errs, e.ApplierEvent.ErrorEvent.Err)
+				}
+			}
+			if tt.expectErr {
+				t.Logf("errors are %v \n", errs)
+				require.Len(t, errs, 1)
+				require.NotNil(t, errs[0])
+				// check if error contains  string
+				assert.Contains(t, errs[0].Error(), tt.expectedString)
+			} else {
+				assert.Len(t, errs, 0)
+			}
+		})
+	}
+}
+
+func newBundle(path string, t *testing.T) document.Bundle {
+	t.Helper()
+	b, err := document.NewBundleByPath(path)
+	require.NoError(t, err)
+	return b
+}
diff --git a/pkg/k8s/applier/apply_options.go b/pkg/k8s/applier/apply_options.go
new file mode 100644
index 000000000..74fbe685b
--- /dev/null
+++ b/pkg/k8s/applier/apply_options.go
@@ -0,0 +1,27 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package applier
+
+import (
+	"time"
+)
+
+// ApplyOptions struct that hold options for apply operation
+type ApplyOptions struct {
+	WaitTimeout time.Duration
+	DryRun      bool
+	Prune       bool
+	BundleName  string
+}
diff --git a/pkg/k8s/applier/errors.go b/pkg/k8s/applier/errors.go
new file mode 100644
index 000000000..7ad5945f2
--- /dev/null
+++ b/pkg/k8s/applier/errors.go
@@ -0,0 +1,37 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package applier
+
+import (
+	"fmt"
+)
+
+// ErrApply returned for not implemented features
+type ErrApply struct {
+	errors []error
+}
+
+func (e ErrApply) Error() string {
+	// TODO make printing more readable here
+	return fmt.Sprintf("Applying of resources to kubernetes cluster has failed, errors are:\n%v", e.errors)
+}
+
+// ErrApplyNilBundle returned when nil bundle is passed to ApplyBundle function
+type ErrApplyNilBundle struct {
+}
+
+func (e ErrApplyNilBundle) Error() string {
+	return "Can not apply nil bundle, airship.kubernetes.Client"
+}
diff --git a/pkg/k8s/applier/fake.go b/pkg/k8s/applier/fake.go
new file mode 100644
index 000000000..9ba318149
--- /dev/null
+++ b/pkg/k8s/applier/fake.go
@@ -0,0 +1,128 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package applier
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/cli-runtime/pkg/resource"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	cliapply "sigs.k8s.io/cli-utils/pkg/apply"
+	applyevent "sigs.k8s.io/cli-utils/pkg/apply/event"
+	"sigs.k8s.io/cli-utils/pkg/apply/poller"
+	"sigs.k8s.io/cli-utils/pkg/kstatus/polling"
+	pollevent "sigs.k8s.io/cli-utils/pkg/kstatus/polling/event"
+	"sigs.k8s.io/cli-utils/pkg/object"
+
+	"opendev.org/airship/airshipctl/pkg/k8s/utils"
+)
+
+// FakeAdaptor is implementation of driver interface
+type FakeAdaptor struct {
+	initErr error
+	events  []applyevent.Event
+}
+
+var _ Driver = FakeAdaptor{}
+
+// NewFakeAdaptor returns a fake driver interface with convience methods for testing
+func NewFakeAdaptor() FakeAdaptor {
+	return FakeAdaptor{}
+}
+
+// Initialize implements driver
+func (fa FakeAdaptor) Initialize(p poller.Poller) error {
+	return fa.initErr
+}
+
+// Run implements driver
+func (fa FakeAdaptor) Run(
+	ctx context.Context,
+	infos []*resource.Info,
+	options cliapply.Options) <-chan applyevent.Event {
+	ch := make(chan applyevent.Event, len(fa.events))
+	defer close(ch)
+	for _, e := range fa.events {
+		ch <- e
+	}
+	return ch
+}
+
+// WithEvents set events for the adaptor
+func (fa FakeAdaptor) WithEvents(events []applyevent.Event) FakeAdaptor {
+	fa.events = events
+	return fa
+}
+
+// WithInitError adds error to Inititialize method
+func (fa FakeAdaptor) WithInitError(err error) FakeAdaptor {
+	fa.initErr = err
+	return fa
+}
+
+// NewFakeApplier returns applier with events you want
+func NewFakeApplier(streams genericclioptions.IOStreams, events []applyevent.Event, f cmdutil.Factory) *Applier {
+	return &Applier{
+		Driver:                NewFakeAdaptor().WithEvents(events),
+		Poller:                &FakePoller{},
+		Factory:               f,
+		ManifestReaderFactory: utils.DefaultManifestReaderFactory,
+	}
+}
+
+// FakePoller returns a poller that sends 2 complete events for now
+// TODO, make these events configurable.
+type FakePoller struct {
+}
+
+// Poll implements poller interface
+func (fp *FakePoller) Poll(ctx context.Context, ids []object.ObjMetadata, opts polling.Options) <-chan pollevent.Event {
+	events := []pollevent.Event{
+		{
+			EventType: pollevent.CompletedEvent,
+			Resource: &pollevent.ResourceStatus{
+				Identifier: object.ObjMetadata{
+					Name:      "test-rc",
+					Namespace: "test",
+					GroupKind: schema.GroupKind{
+						Group: "v1",
+						Kind:  "ReplicationController",
+					},
+				},
+			},
+		},
+		{
+			EventType: pollevent.CompletedEvent,
+			Resource: &pollevent.ResourceStatus{
+				Identifier: object.ObjMetadata{
+					Name:      "airshipit-test-bundle-4bf1e4a",
+					Namespace: "airshipit",
+					GroupKind: schema.GroupKind{
+						Group: "v1",
+						Kind:  "ConfigMap",
+					},
+				},
+			},
+		},
+	}
+	ch := make(chan pollevent.Event, len(events))
+	defer close(ch)
+	for _, e := range events {
+		ch <- e
+	}
+	return ch
+}
diff --git a/pkg/k8s/applier/testdata/kubeconfig.yaml b/pkg/k8s/applier/testdata/kubeconfig.yaml
new file mode 100644
index 000000000..967864a76
--- /dev/null
+++ b/pkg/k8s/applier/testdata/kubeconfig.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1Ea3lPVEUzTURNd09Wb1hEVEk1TURreU5qRTNNRE13T1Zvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTUZyCkdxM0kyb2dZci81Y01Udy9Na1pORTNWQURzdEdyU240WjU2TDhPUGhMcUhDN2t1dno2dVpES3dCSGtGeTBNK2MKRXIzd2piUGE1aTV5NmkyMGtxSHBVMjdPZTA0dzBXV2s4N0RSZVlWaGNoZVJHRXoraWt3SndIcGRmMjJVemZNKwpkSDBzaUhuMVd6UnovYk4za3hMUzJlMnZ2U1Y3bmNubk1YRUd4OXV0MUY0NThHeWxxdmxXTUlWMzg5Q2didXFDCkcwcFdiMTBLM0RVZWdiT25Xa1FmSm5sTWRRVVZDUVdZZEZaaklrcWtkWi9hVTRobkNEV01oZXNWRnFNaDN3VVAKczhQay9BNWh1ZFFPbnFRNDVIWXZLdjZ5RjJWcDUyWExBRUx3NDJ4aVRKZlh0V1h4eHR6cU4wY1lyL2VxeS9XMQp1YVVGSW5xQjFVM0JFL1oxbmFrQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFKUUVKQVBLSkFjVDVuK3dsWGJsdU9mS0J3c2gKZTI4R1c5R2QwM0N0NGF3RzhzMXE1ZHNua2tpZmVTUENHVFZ1SXF6UTZDNmJaSk9SMDMvVEl5ejh6NDJnaitDVApjWUZXZkltM2RKTnpRL08xWkdySXZZNWdtcWJtWDlpV0JaU24rRytEOGxubzd2aGMvY0tBRFR5OTMvVU92MThuCkdhMnIrRGJJcHcyTWVBVEl2elpxRS9RWlVSQ25DMmdjUFhTVzFqN2h4R3o1a3ZNcGVDZTdQYVUvdVFvblVHSWsKZ2t6ZzI4NHQvREhUUzc4N1V1SUg5cXBaV09yTFNMOGFBeUxQUHhWSXBteGZmbWRETE9TS2VUemRlTmxoSitUMwowQlBVaHBQTlJBNTNJN0hRQjhVUDR2elNONTkzZ1VFbVlFQ2Jic2RYSzB6ZVR6SDdWWHR2Zmd5WTVWWT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+    server: https://10.0.1.7:6443
+  name: kubernetes_target
+contexts:
+- context:
+    cluster: kubernetes_target
+    user: kubernetes-admin
+  name: kubernetes-admin@kubernetes
+current-context: ""
+kind: Config
+preferences: {}
+users:
+- name: kubernetes-admin
+  user:
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJQXhEdzk2RUY4SXN3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T1RBNU1qa3hOekF6TURsYUZ3MHlNREE1TWpneE56QXpNVEphTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXV6R0pZdlBaNkRvaTQyMUQKSzhXSmFaQ25OQWQycXo1cC8wNDJvRnpRUGJyQWd6RTJxWVZrek9MOHhBVmVSN1NONXdXb1RXRXlGOEVWN3JyLwo0K0hoSEdpcTVQbXF1SUZ5enpuNi9JWmM4alU5eEVmenZpa2NpckxmVTR2UlhKUXdWd2dBU05sMkFXQUloMmRECmRUcmpCQ2ZpS1dNSHlqMFJiSGFsc0J6T3BnVC9IVHYzR1F6blVRekZLdjJkajVWMU5rUy9ESGp5UlJKK0VMNlEKQlltR3NlZzVQNE5iQzllYnVpcG1NVEFxL0p1bU9vb2QrRmpMMm5acUw2Zkk2ZkJ0RjVPR2xwQ0IxWUo4ZnpDdApHUVFaN0hUSWJkYjJ0cDQzRlZPaHlRYlZjSHFUQTA0UEoxNSswV0F5bVVKVXo4WEE1NDRyL2J2NzRKY0pVUkZoCmFyWmlRd0lEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFMMmhIUmVibEl2VHJTMFNmUVg1RG9ueVVhNy84aTg1endVWApSd3dqdzFuS0U0NDJKbWZWRGZ5b0hRYUM4Ti9MQkxyUXM0U0lqU1JYdmFHU1dSQnRnT1RRV21Db1laMXdSbjdwCndDTXZQTERJdHNWWm90SEZpUFl2b1lHWFFUSXA3YlROMmg1OEJaaEZ3d25nWUovT04zeG1rd29IN1IxYmVxWEYKWHF1TTluekhESk41VlZub1lQR09yRHMwWlg1RnNxNGtWVU0wVExNQm9qN1ZIRDhmU0E5RjRYNU4yMldsZnNPMAo4aksrRFJDWTAyaHBrYTZQQ0pQS0lNOEJaMUFSMG9ZakZxT0plcXpPTjBqcnpYWHh4S2pHVFVUb1BldVA5dCtCCjJOMVA1TnI4a2oxM0lrend5Q1NZclFVN09ZM3ltZmJobHkrcXZxaFVFa014MlQ1SkpmQT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+    client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBdXpHSll2UFo2RG9pNDIxREs4V0phWkNuTkFkMnF6NXAvMDQyb0Z6UVBickFnekUyCnFZVmt6T0w4eEFWZVI3U041d1dvVFdFeUY4RVY3cnIvNCtIaEhHaXE1UG1xdUlGeXp6bjYvSVpjOGpVOXhFZnoKdmlrY2lyTGZVNHZSWEpRd1Z3Z0FTTmwyQVdBSWgyZERkVHJqQkNmaUtXTUh5ajBSYkhhbHNCek9wZ1QvSFR2MwpHUXpuVVF6Rkt2MmRqNVYxTmtTL0RIanlSUkorRUw2UUJZbUdzZWc1UDROYkM5ZWJ1aXBtTVRBcS9KdW1Pb29kCitGakwyblpxTDZmSTZmQnRGNU9HbHBDQjFZSjhmekN0R1FRWjdIVEliZGIydHA0M0ZWT2h5UWJWY0hxVEEwNFAKSjE1KzBXQXltVUpVejhYQTU0NHIvYnY3NEpjSlVSRmhhclppUXdJREFRQUJBb0lCQVFDU0pycjlaeVpiQ2dqegpSL3VKMFZEWCt2aVF4c01BTUZyUjJsOE1GV3NBeHk1SFA4Vk4xYmc5djN0YUVGYnI1U3hsa3lVMFJRNjNQU25DCm1uM3ZqZ3dVQWlScllnTEl5MGk0UXF5VFBOU1V4cnpTNHRxTFBjM3EvSDBnM2FrNGZ2cSsrS0JBUUlqQnloamUKbnVFc1JpMjRzT3NESlM2UDE5NGlzUC9yNEpIM1M5bFZGbkVuOGxUR2c0M1kvMFZoMXl0cnkvdDljWjR5ZUNpNwpjMHFEaTZZcXJZaFZhSW9RRW1VQjdsbHRFZkZzb3l4VDR6RTE5U3pVbkRoMmxjYTF1TzhqcmI4d2xHTzBoQ2JyClB1R1l2WFFQa3Q0VlNmalhvdGJ3d2lBNFRCVERCRzU1bHp6MmNKeS9zSS8zSHlYbEMxcTdXUmRuQVhhZ1F0VzkKOE9DZGRkb0JBb0dCQU5NcUNtSW94REtyckhZZFRxT1M1ZFN4cVMxL0NUN3ZYZ0pScXBqd2Y4WHA2WHo0KzIvTAozVXFaVDBEL3dGTkZkc1Z4eFYxMnNYMUdwMHFWZVlKRld5OVlCaHVSWGpTZ0ZEWldSY1Z1Y01sNVpPTmJsbmZGCjVKQ0xnNXFMZ1g5VTNSRnJrR3A0R241UDQxamg4TnhKVlhzZG5xWE9xNTFUK1RRT1UzdkpGQjc1QW9HQkFPTHcKalp1cnZtVkZyTHdaVGgvRDNpWll5SVV0ZUljZ2NKLzlzbTh6L0pPRmRIbFd4dGRHUFVzYVd1MnBTNEhvckFtbgpqTm4vSTluUXd3enZ3MWUzVVFPbUhMRjVBczk4VU5hbk5TQ0xNMW1yaXZHRXJ1VHFnTDM1bU41eFZPdTUxQU5JCm4yNkFtODBJT2JDeEtLa0R0ZXJSaFhHd3g5c1pONVJCbG9VRThZNGJBb0dBQ3ZsdVhMZWRxcng5VkE0bDNoNXUKVDJXRVUxYjgxZ1orcmtRc1I1S0lNWEw4cllBTElUNUpHKzFuendyN3BkaEFXZmFWdVV2SDRhamdYT0h6MUs5aQpFODNSVTNGMG9ldUg0V01PY1RwU0prWm0xZUlXcWRiaEVCb1FGdUlWTXRib1BsV0d4ZUhFRHJoOEtreGp4aThSCmdEcUQyajRwY1IzQ0g5QjJ5a0lqQjVFQ2dZRUExc0xXLys2enE1c1lNSm14K1JXZThhTXJmL3pjQnVTSU1LQWgKY0dNK0wwMG9RSHdDaUU4TVNqcVN1ajV3R214YUFuanhMb3ZwSFlRV1VmUEVaUW95UE1YQ2VhRVBLOU4xbk8xMwp0V2lHRytIZkIxaU5PazFCc0lhNFNDbndOM1FRVTFzeXBaeEgxT3hueS9LYmkvYmEvWEZ5VzNqMGFUK2YvVWxrCmJGV1ZVdWtDZ1lFQTBaMmRTTFlmTjV5eFNtYk5xMWVqZXdWd1BjRzQxR2hQclNUZEJxdHFac1doWGE3aDdLTWEKeHdvamh5SXpnTXNyK2tXODdlajhDQ2h0d21sQ1p5QU92QmdOZytncnJ1cEZLM3FOSkpKeU9YREdHckdpbzZmTQp5aXB3Q2tZVGVxRThpZ1J6UkI5QkdFUGY4eVpjMUtwdmZhUDVhM0lRZmxiV0czbGpUemNNZVZjPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
diff --git a/pkg/k8s/applier/testdata/source_bundle/kustomization.yaml b/pkg/k8s/applier/testdata/source_bundle/kustomization.yaml
new file mode 100644
index 000000000..72ab8548e
--- /dev/null
+++ b/pkg/k8s/applier/testdata/source_bundle/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+  - replicationcontroller.yaml
diff --git a/pkg/k8s/applier/testdata/source_bundle/replicationcontroller.yaml b/pkg/k8s/applier/testdata/source_bundle/replicationcontroller.yaml
new file mode 100644
index 000000000..1e39abdd8
--- /dev/null
+++ b/pkg/k8s/applier/testdata/source_bundle/replicationcontroller.yaml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: test-rc
+  namespace: test
+  annotations:
+    airshipit.org/initinfra: "workflow"
+  labels:
+    name: test-rc
+    airshipit.org/initinfra: "workflow"
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: test-rc
+    spec:
+      containers:
+        - name: test-rc
+          image: nginx
+          ports:
+          - containerPort: 80
diff --git a/pkg/k8s/applier/testdata/two_cm_bundle/kustomization.yaml b/pkg/k8s/applier/testdata/two_cm_bundle/kustomization.yaml
new file mode 100644
index 000000000..6e0356d1a
--- /dev/null
+++ b/pkg/k8s/applier/testdata/two_cm_bundle/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+  - resources.yaml
\ No newline at end of file
diff --git a/pkg/k8s/applier/testdata/two_cm_bundle/resources.yaml b/pkg/k8s/applier/testdata/two_cm_bundle/resources.yaml
new file mode 100644
index 000000000..39b33f562
--- /dev/null
+++ b/pkg/k8s/applier/testdata/two_cm_bundle/resources.yaml
@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: test-rc
+  namespace: test
+  annotations:
+    airshipit.org/initinfra: "workflow"
+  labels:
+    name: test-rc
+    airshipit.org/initinfra: "workflow"
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: test-rc
+    spec:
+      containers:
+        - name: test-rc
+          image: nginx
+          ports:
+          - containerPort: 80
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: first-map
+  namespace: default
+  labels:
+    cli-utils.sigs.k8s.io/inventory-id: "some id"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: second-map
+  namespace: default
+  labels:
+    cli-utils.sigs.k8s.io/inventory-id: "some id"
+
+