From f9378e74b03e75c00cf9079a331aaf27446ddfa0 Mon Sep 17 00:00:00 2001
From: bijayasharma <vetbijaya@gmail.com>
Date: Wed, 11 Nov 2020 14:02:23 -0500
Subject: [PATCH] Add phase tree command in package module
* This commit is the smaller patchset from earlier work:
https://review.opendev.org/#/c/750449/
* This is the first series of patchset
Change-Id: Ib1ff74eba65de7c7c59cf8f4fd26b15e388ba368
Signed-off-by: bijayasharma <vetbijaya@gmail.com>
Relates-To: #296
---
pkg/document/constants.go | 3 +
.../testdata/no_plan_site/metadata.yaml | 4 +
.../no_plan_site/phases/kustomization.yaml | 0
.../hostgenerator/host-generation.yaml | 10 +
.../hostgenerator/kustomization.yaml | 4 +
.../workers-targetphase/kustomization.yaml | 6 +
.../nodes/kustomization.yaml | 5 +
pkg/document/tree.go | 186 +++++++++++
pkg/document/tree_test.go | 297 ++++++++++++++++++
pkg/phase/command.go | 55 ++++
pkg/phase/command_test.go | 67 ++++
11 files changed, 637 insertions(+)
create mode 100644 pkg/document/testdata/no_plan_site/metadata.yaml
create mode 100644 pkg/document/testdata/no_plan_site/phases/kustomization.yaml
create mode 100644 pkg/document/testdata/workers-targetphase/hostgenerator/host-generation.yaml
create mode 100644 pkg/document/testdata/workers-targetphase/hostgenerator/kustomization.yaml
create mode 100644 pkg/document/testdata/workers-targetphase/kustomization.yaml
create mode 100644 pkg/document/testdata/workers-targetphase/nodes/kustomization.yaml
create mode 100644 pkg/document/tree.go
create mode 100644 pkg/document/tree_test.go
diff --git a/pkg/document/constants.go b/pkg/document/constants.go
index 6eee1d4e0..e18570a52 100644
--- a/pkg/document/constants.go
+++ b/pkg/document/constants.go
@@ -37,3 +37,6 @@ const (
ClusterctlMetadataVersion = "v1alpha3"
ClusterctlMetadataGroup = "clusterctl.cluster.x-k8s.io"
)
+
+// KustomizationFile is used for kustomization file
+const KustomizationFile = "kustomization.yaml"
diff --git a/pkg/document/testdata/no_plan_site/metadata.yaml b/pkg/document/testdata/no_plan_site/metadata.yaml
new file mode 100644
index 000000000..01eb92b8d
--- /dev/null
+++ b/pkg/document/testdata/no_plan_site/metadata.yaml
@@ -0,0 +1,4 @@
+inventory:
+ path: "manifests/site/inventory"
+phase:
+ path: "no_plan_site/phases"
\ No newline at end of file
diff --git a/pkg/document/testdata/no_plan_site/phases/kustomization.yaml b/pkg/document/testdata/no_plan_site/phases/kustomization.yaml
new file mode 100644
index 000000000..e69de29bb
diff --git a/pkg/document/testdata/workers-targetphase/hostgenerator/host-generation.yaml b/pkg/document/testdata/workers-targetphase/hostgenerator/host-generation.yaml
new file mode 100644
index 000000000..1ef2d9648
--- /dev/null
+++ b/pkg/document/testdata/workers-targetphase/hostgenerator/host-generation.yaml
@@ -0,0 +1,10 @@
+# Site-level, phase-specific lists of hosts to generate
+# This is used by the hostgenerator-m3 function to narrow down the site-level
+# host-catalogue to just the hosts needed for a particular phase.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+ name: host-generation-catalogue
+hosts:
+ m3:
+ - node03
\ No newline at end of file
diff --git a/pkg/document/testdata/workers-targetphase/hostgenerator/kustomization.yaml b/pkg/document/testdata/workers-targetphase/hostgenerator/kustomization.yaml
new file mode 100644
index 000000000..2a82be926
--- /dev/null
+++ b/pkg/document/testdata/workers-targetphase/hostgenerator/kustomization.yaml
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+ - host-generation.yaml
diff --git a/pkg/document/testdata/workers-targetphase/kustomization.yaml b/pkg/document/testdata/workers-targetphase/kustomization.yaml
new file mode 100644
index 000000000..33bb7d64c
--- /dev/null
+++ b/pkg/document/testdata/workers-targetphase/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+ - nodes
+generators:
+ - hostgenerator
diff --git a/pkg/document/testdata/workers-targetphase/nodes/kustomization.yaml b/pkg/document/testdata/workers-targetphase/nodes/kustomization.yaml
new file mode 100644
index 000000000..585f3f949
--- /dev/null
+++ b/pkg/document/testdata/workers-targetphase/nodes/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+commonLabels:
+ airshipit.org/k8s-role: controlplane-host
diff --git a/pkg/document/tree.go b/pkg/document/tree.go
new file mode 100644
index 000000000..38bb7ee62
--- /dev/null
+++ b/pkg/document/tree.go
@@ -0,0 +1,186 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package document
+
+import (
+ "fmt"
+ "io"
+ "path/filepath"
+
+ "sigs.k8s.io/kustomize/api/types"
+
+ "opendev.org/airship/airshipctl/pkg/fs"
+)
+
+// KustomNode is used to create name and data to display tree structure
+type KustomNode struct {
+ Name string // name used for display purposes (cli)
+ Data string // this could be a Kustomization object, or a string containing a file path
+ Children []KustomNode
+ Writer io.Writer
+}
+
+// BuildKustomTree creates a tree based on entrypoint
+func BuildKustomTree(entrypoint string, writer io.Writer, manifestsDir string) (KustomNode, error) {
+ fs := fs.NewDocumentFs()
+ name, err := filepath.Rel(manifestsDir, entrypoint)
+ if err != nil {
+ name = entrypoint
+ }
+ root := KustomNode{
+ Name: name,
+ Data: entrypoint,
+ Children: []KustomNode{},
+ Writer: writer,
+ }
+
+ resMap, err := MakeResMap(fs, entrypoint)
+ if err != nil {
+ return KustomNode{}, err
+ }
+
+ for sourceType, sources := range resMap {
+ n := KustomNode{
+ Name: sourceType,
+ Writer: writer,
+ }
+
+ for _, s := range sources {
+ if !fs.IsDir(s) {
+ name, err := filepath.Rel(manifestsDir, s)
+ if err != nil {
+ name = s
+ }
+ n.Children = append(n.Children, KustomNode{
+ Name: name,
+ Data: s,
+ })
+ } else {
+ s = filepath.Join(s, KustomizationFile)
+ child, err := BuildKustomTree(s, writer, "")
+ if err != nil {
+ return KustomNode{}, err
+ }
+ n.Children = append(n.Children, child)
+ }
+ }
+ root.Children = append(root.Children, n)
+ }
+ return root, nil
+}
+
+//MakeResMap creates resmap based of kustomize types
+func MakeResMap(fs fs.FileSystem, kfile string) (map[string][]string, error) {
+ if fs == nil {
+ return nil, fmt.Errorf("received nil filesystem")
+ }
+ bytes, err := fs.ReadFile(kfile)
+ if err != nil {
+ return nil, err
+ }
+
+ k := types.Kustomization{}
+ err = k.Unmarshal(bytes)
+ if err != nil {
+ return nil, err
+ }
+ basedir := filepath.Dir(kfile)
+ var resMap = make(map[string][]string)
+ for _, p := range k.Resources {
+ path := filepath.Join(basedir, p)
+ resMap["Resources"] = append(resMap["Resources"], path)
+ }
+
+ for _, p := range k.Crds {
+ path := filepath.Join(basedir, p)
+ resMap["Crds"] = append(resMap["Crds"], path)
+ }
+
+ buildConfigMapAndSecretGenerator(k, basedir, resMap)
+
+ for _, p := range k.Generators {
+ path := filepath.Join(basedir, p)
+ resMap["Generators"] = append(resMap["Generators"], path)
+ }
+
+ for _, p := range k.Transformers {
+ path := filepath.Join(basedir, p)
+ resMap["Transformers"] = append(resMap["Transformers"], path)
+ }
+
+ return resMap, nil
+}
+
+func buildConfigMapAndSecretGenerator(k types.Kustomization, basedir string, resMap map[string][]string) {
+ for _, p := range k.SecretGenerator {
+ for _, s := range p.FileSources {
+ path := filepath.Join(basedir, s)
+ resMap["SecretGenerator"] = append(resMap["SecretGenerator"], path)
+ }
+ }
+ for _, p := range k.ConfigMapGenerator {
+ for _, s := range p.FileSources {
+ path := filepath.Join(basedir, s)
+ resMap["ConfigMapGenerator"] = append(resMap["ConfigMapGenerator"], path)
+ }
+ }
+}
+
+// PrintTree prints tree view of phase
+func (k KustomNode) PrintTree(prefix string) {
+ if prefix == "" {
+ basedir := filepath.Dir(k.Name)
+ dir := filepath.Base(basedir)
+ fmt.Fprintf(k.Writer, "%s [%s]\n", dir, basedir)
+ }
+ for i, child := range k.Children {
+ var subprefix string
+ knodes := GetKustomChildren(child)
+ if len(knodes) > 0 {
+ // we found a kustomize file, so print the subtree name first
+ if i == len(k.Children)-1 {
+ fmt.Fprintf(k.Writer, "%s└── %s\n", prefix, child.Name)
+ subprefix = " "
+ } else {
+ fmt.Fprintf(k.Writer, "%s├── %s\n", prefix, child.Name)
+ subprefix = "│ "
+ }
+ }
+ for j, c := range knodes {
+ bd := filepath.Dir(c.Name)
+ d := filepath.Base(bd)
+ name := fmt.Sprintf("%s [%s]", d, bd)
+
+ if j == len(knodes)-1 {
+ fmt.Printf("%s%s└── %s\n", prefix, subprefix, name)
+ c.PrintTree(fmt.Sprintf("%s%s ", prefix, subprefix))
+ } else {
+ fmt.Printf("%s%s├── %s\n", prefix, subprefix, name)
+ c.PrintTree(fmt.Sprintf("%s%s│ ", prefix, subprefix))
+ }
+ }
+ }
+}
+
+// GetKustomChildren returns children nodes of kustomnode
+func GetKustomChildren(k KustomNode) []KustomNode {
+ nodes := []KustomNode{}
+ for _, c := range k.Children {
+ if len(c.Children) > 0 {
+ nodes = append(nodes, c)
+ }
+ }
+ return nodes
+}
diff --git a/pkg/document/tree_test.go b/pkg/document/tree_test.go
new file mode 100644
index 000000000..105411f52
--- /dev/null
+++ b/pkg/document/tree_test.go
@@ -0,0 +1,297 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package document_test
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "os"
+ "testing"
+
+ "bufio"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "opendev.org/airship/airshipctl/pkg/document"
+ "opendev.org/airship/airshipctl/pkg/fs"
+)
+
+func KustomNodeTestdata(writer io.Writer) document.KustomNode {
+ return document.KustomNode{
+ Name: "workers-targetphase/kustomization.yaml",
+ Data: "testdata/workers-targetphase/kustomization.yaml",
+ Children: []document.KustomNode{
+ {
+ Name: "Resources",
+ Data: "",
+ Children: []document.KustomNode{
+ {
+ Name: "workers-targetphase/nodes/kustomization.yaml",
+ Data: "testdata/workers-targetphase/nodes/kustomization.yaml",
+ Children: []document.KustomNode{},
+ },
+ },
+ Writer: writer,
+ },
+ {
+ Name: "Generators",
+ Data: "",
+ Children: []document.KustomNode{
+ {
+ Name: "workers-targetphase/hostgenerator/kustomization.yaml",
+ Data: "testdata/workers-targetphase/hostgenerator/kustomization.yaml",
+ Children: []document.KustomNode{{
+ Name: "Resources",
+ Data: "",
+ Children: []document.KustomNode{{
+ Name: "workers-targetphase/hostgenerator/host-generation.yaml",
+ Data: "testdata/workers-targetphase/hostgenerator/host-generation.yaml",
+ },
+ },
+ Writer: writer,
+ },
+ },
+ },
+ },
+ Writer: writer,
+ },
+ },
+ Writer: writer,
+ }
+}
+
+func TestBuildKustomTree(t *testing.T) {
+ var b bytes.Buffer
+ w := bufio.NewWriter(&b)
+ type args struct {
+ entrypoint string
+ }
+ tests := []struct {
+ name string
+ args func(t *testing.T) args
+ want1 document.KustomNode
+ errContains string
+ }{
+ {
+ name: "success build tree",
+ args: func(t *testing.T) args {
+ return args{entrypoint: "testdata/workers-targetphase/kustomization.yaml"}
+ },
+ want1: KustomNodeTestdata(w),
+ },
+ {
+ name: "entrypoint doesn't exist",
+ args: func(t *testing.T) args {
+ return args{entrypoint: "tdata/kustomization.yaml"}
+ },
+ want1: KustomNodeTestdata(w),
+ errContains: "no such file or directory",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tArgs := tt.args(t)
+ manifestsDir := "testdata"
+ got1, actualErr := document.BuildKustomTree(tArgs.entrypoint, w, manifestsDir)
+ if tt.errContains != "" {
+ require.Error(t, actualErr)
+ assert.Contains(t, actualErr.Error(), tt.errContains)
+ } else {
+ require.NoError(t, actualErr)
+ assert.Equal(t, got1.Name, tt.want1.Name)
+ assert.Equal(t, len(got1.Children), len(tt.want1.Children))
+ }
+ })
+ }
+}
+
+func Test_makeResMap(t *testing.T) {
+ type args struct {
+ kfile string
+ fs fs.FileSystem
+ }
+ tests := []struct {
+ name string
+ args func(t *testing.T) args
+
+ want1 map[string][]string
+ errContains string
+ }{
+ {
+ args: func(t *testing.T) args {
+ return args{kfile: "testdata/workers-targetphase/kustomization.yaml", fs: fs.NewDocumentFs()}
+ },
+ name: "success resmap",
+ want1: map[string][]string{
+ "Generators": {
+ "testdata/workers-targetphase/hostgenerator",
+ },
+ "Resources": {
+ "testdata/workers-targetphase/nodes",
+ },
+ },
+ },
+ {
+ args: func(t *testing.T) args {
+ return args{kfile: "testdata/no_plan_site/phases/kustomization.yaml"}
+ },
+ name: "nil case",
+ want1: map[string][]string{},
+ errContains: "received nil filesystem",
+ },
+ {
+ args: func(t *testing.T) args {
+ return args{kfile: "t/workers-targetphase/kustomization.yaml", fs: fs.NewDocumentFs()}
+ },
+ name: "fail resmap,entrypoint not found",
+ want1: map[string][]string{
+ "Resources": {
+ "testdata/workers-targetphase/nodes",
+ },
+ },
+ errContains: "no such file or directory",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tArgs := tt.args(t)
+
+ got1, actualErr := document.MakeResMap(tArgs.fs, tArgs.kfile)
+ if tt.errContains != "" {
+ require.Error(t, actualErr)
+ assert.Contains(t, actualErr.Error(), tt.errContains)
+ } else {
+ require.NoError(t, actualErr)
+ assert.Equal(t, got1, tt.want1)
+ }
+ })
+ }
+}
+
+func TestKustomNode_PrintTree(t *testing.T) {
+ var b bytes.Buffer
+ writer := bufio.NewWriter(&b)
+ type args struct {
+ prefix string
+ }
+ tests := []struct {
+ name string
+ init func(t *testing.T) document.KustomNode
+ want string
+
+ args func(t *testing.T) args
+ }{
+ {
+ name: "valid print tree",
+ args: func(t *testing.T) args {
+ return args{prefix: ""}
+ },
+ init: func(t *testing.T) document.KustomNode {
+ return KustomNodeTestdata(writer)
+ },
+ want: " └── hostgenerator [workers-targetphase/hostgenerator]\n",
+ },
+ }
+ rescueStdout := os.Stdout
+ r, w, err := os.Pipe()
+ if err != nil {
+ require.Error(t, err)
+ }
+ os.Stdout = w
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tArgs := tt.args(t)
+
+ receiver := tt.init(t)
+ receiver.PrintTree(tArgs.prefix)
+ w.Close()
+ out, err := ioutil.ReadAll(r)
+ if err != nil {
+ require.Error(t, err)
+ }
+ os.Stdout = rescueStdout
+ assert.Equal(t, string(out), tt.want)
+ })
+ }
+}
+
+func Test_getKustomChildren(t *testing.T) {
+ type args struct {
+ k document.KustomNode
+ }
+ tests := []struct {
+ name string
+ args func(t *testing.T) args
+
+ want1 []document.KustomNode
+ }{
+ {
+ name: "success getkustomchildren",
+ args: func(t *testing.T) args {
+ return args{k: document.KustomNode{
+ Name: "Generators",
+ Data: "",
+ Children: []document.KustomNode{
+ {
+ Name: "workers-targetphase/hostgenerator/kustomization.yaml",
+ Data: "testdata/workers-targetphase/hostgenerator/kustomization.yaml",
+ Children: []document.KustomNode{{
+ Name: "workers-targetphase/hostgenerator/host-generation.yaml",
+ Data: "testdata/workers-targetphase/hostgenerator/host-generation.yaml",
+ },
+ },
+ },
+ },
+ },
+ }
+ },
+ want1: []document.KustomNode{
+ {
+ Name: "workers-targetphase/hostgenerator/kustomization.yaml",
+ Data: "testdata/workers-targetphase/hostgenerator/kustomization.yaml",
+ Children: []document.KustomNode{{
+ Name: "workers-targetphase/hostgenerator/host-generation.yaml",
+ Data: "testdata/workers-targetphase/hostgenerator/host-generation.yaml",
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "no children nodes",
+ args: func(t *testing.T) args {
+ return args{k: document.KustomNode{
+ Name: "Transformers",
+ Data: "",
+ }}
+ },
+ want1: []document.KustomNode{},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tArgs := tt.args(t)
+
+ got1 := document.GetKustomChildren(tArgs.k)
+ assert.Equal(t, got1, tt.want1)
+ })
+ }
+}
diff --git a/pkg/phase/command.go b/pkg/phase/command.go
index a2c915cdd..31bab8278 100644
--- a/pkg/phase/command.go
+++ b/pkg/phase/command.go
@@ -16,9 +16,13 @@ package phase
import (
"io"
+ "os"
+ "path/filepath"
+ "strings"
"time"
"opendev.org/airship/airshipctl/pkg/config"
+ "opendev.org/airship/airshipctl/pkg/document"
"opendev.org/airship/airshipctl/pkg/phase/ifc"
)
@@ -84,3 +88,54 @@ func (c *PlanCommand) RunE() error {
return PrintPlan(plan, c.Writer)
}
+
+// TreeCommand plan command
+type TreeCommand struct {
+ Factory config.Factory
+ PhaseID ifc.ID
+ Writer io.Writer
+ Argument string
+}
+
+// RunE runs the phase tree command
+func (c *TreeCommand) RunE() error {
+ var entrypoint string
+ cfg, err := c.Factory()
+ if err != nil {
+ return err
+ }
+
+ helper, err := NewHelper(cfg)
+ if err != nil {
+ return err
+ }
+
+ client := NewClient(helper)
+ var manifestsDir string
+ // check if its a relative path
+ if _, err = os.Stat(c.Argument); err == nil {
+ // capture manifests directory from phase relative path
+ manifestsDir = strings.SplitAfter(c.Argument, "/manifests")[0]
+ entrypoint = filepath.Join(c.Argument, document.KustomizationFile)
+ } else {
+ c.PhaseID.Name = c.Argument
+ manifestsDir = filepath.Join(helper.TargetPath(), helper.PhaseRepoDir())
+ var phase ifc.Phase
+ phase, err = client.PhaseByID(c.PhaseID)
+ if err != nil {
+ return err
+ }
+ var rootPath string
+ rootPath, err = phase.DocumentRoot()
+ if err != nil {
+ return err
+ }
+ entrypoint = filepath.Join(rootPath, document.KustomizationFile)
+ }
+ t, err := document.BuildKustomTree(entrypoint, c.Writer, manifestsDir)
+ if err != nil {
+ return err
+ }
+ t.PrintTree("")
+ return nil
+}
diff --git a/pkg/phase/command_test.go b/pkg/phase/command_test.go
index f143e76d0..6ddf3e835 100644
--- a/pkg/phase/command_test.go
+++ b/pkg/phase/command_test.go
@@ -169,3 +169,70 @@ func TestPlanCommand(t *testing.T) {
})
}
}
+
+func TestTreeCommand(t *testing.T) {
+ tests := []struct {
+ name string
+ errContains string
+ factory config.Factory
+ }{
+ {
+ name: "Error config factory",
+ factory: func() (*config.Config, error) {
+ return nil, fmt.Errorf(testFactoryErr)
+ },
+ errContains: testFactoryErr,
+ },
+ {
+ name: "Error new helper",
+ factory: func() (*config.Config, error) {
+ return &config.Config{
+ CurrentContext: "does not exist",
+ Contexts: make(map[string]*config.Context),
+ }, nil
+ },
+ errContains: testNewHelperErr,
+ },
+ {
+ name: "Error phase by id",
+ factory: func() (*config.Config, error) {
+ conf := config.NewConfig()
+ conf.Manifests = map[string]*config.Manifest{
+ "manifest": {
+ MetadataPath: "broken_metadata.yaml",
+ TargetPath: "testdata",
+ PhaseRepositoryName: config.DefaultTestPhaseRepo,
+ Repositories: map[string]*config.Repository{
+ config.DefaultTestPhaseRepo: {
+ URLString: "",
+ },
+ },
+ },
+ }
+ conf.CurrentContext = "context"
+ conf.Contexts = map[string]*config.Context{
+ "context": {
+ Manifest: "manifest",
+ },
+ }
+ return conf, nil
+ },
+ errContains: testNoBundlePath,
+ },
+ }
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ command := phase.TreeCommand{
+ Factory: tt.factory,
+ }
+ err := command.RunE()
+ if tt.errContains != "" {
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), tt.errContains)
+ } else {
+ assert.NoError(t, err)
+ }
+ })
+ }
+}