From 4903ecd30b71fe1220296f547a4061a6d12e15f4 Mon Sep 17 00:00:00 2001 From: Tristan Cacqueray Date: Thu, 9 Apr 2020 12:17:13 +0000 Subject: [PATCH] dhall-diff: add new job This change adds a new dhall-diff job to verify generated config is idempotent. Change-Id: I96a335dc78c4fa74564b854997433e5be0b5e633 --- doc/source/general-jobs.rst | 1 + doc/source/general-roles.rst | 2 ++ playbooks/dhall/diff.yaml | 3 ++ playbooks/dhall/prepare.yaml | 25 +++++++++++++++ roles/ensure-dhall/README.rst | 10 ++++++ roles/ensure-dhall/defaults/main.yaml | 9 ++++++ roles/ensure-dhall/tasks/main.yaml | 33 ++++++++++++++++++++ roles/render-diff/README.rst | 15 +++++++++ roles/render-diff/defaults/main.yaml | 4 +++ roles/render-diff/tasks/main.yaml | 38 +++++++++++++++++++++++ test-playbooks/dhall/setup-project.yaml | 23 ++++++++++++++ test-playbooks/dhall/test-dhall-diff.yaml | 31 ++++++++++++++++++ zuul-tests.d/dhall.yaml | 17 ++++++++++ zuul.d/general-jobs.yaml | 21 +++++++++++++ 14 files changed, 232 insertions(+) create mode 100644 playbooks/dhall/diff.yaml create mode 100644 playbooks/dhall/prepare.yaml create mode 100644 roles/ensure-dhall/README.rst create mode 100644 roles/ensure-dhall/defaults/main.yaml create mode 100644 roles/ensure-dhall/tasks/main.yaml create mode 100644 roles/render-diff/README.rst create mode 100644 roles/render-diff/defaults/main.yaml create mode 100644 roles/render-diff/tasks/main.yaml create mode 100644 test-playbooks/dhall/setup-project.yaml create mode 100644 test-playbooks/dhall/test-dhall-diff.yaml create mode 100644 zuul-tests.d/dhall.yaml diff --git a/doc/source/general-jobs.rst b/doc/source/general-jobs.rst index 7803f82b7..56d6e4c50 100644 --- a/doc/source/general-jobs.rst +++ b/doc/source/general-jobs.rst @@ -2,6 +2,7 @@ General Purpose Jobs ==================== .. zuul:autojob:: dco-license +.. zuul:autojob:: dhall-diff .. zuul:autojob:: unittests .. zuul:autojob:: markdownlint .. zuul:autojob:: multinode diff --git a/doc/source/general-roles.rst b/doc/source/general-roles.rst index fa928d5a7..9e8ceacc8 100644 --- a/doc/source/general-roles.rst +++ b/doc/source/general-roles.rst @@ -15,6 +15,7 @@ General Purpose Roles .. zuul:autorole:: emit-job-header .. zuul:autorole:: enable-netconsole .. zuul:autorole:: ensure-bazelisk +.. zuul:autorole:: ensure-dhall .. zuul:autorole:: ensure-dstat-graph .. zuul:autorole:: ensure-markdownlint .. zuul:autorole:: fetch-markdownlint @@ -33,6 +34,7 @@ General Purpose Roles .. zuul:autorole:: remove-build-sshkey .. zuul:autorole:: remove-gpgkey .. zuul:autorole:: remove-sshkey +.. zuul:autorole:: render-diff .. zuul:autorole:: revoke-sudo .. zuul:autorole:: run-dstat .. zuul:autorole:: sign-artifacts diff --git a/playbooks/dhall/diff.yaml b/playbooks/dhall/diff.yaml new file mode 100644 index 000000000..9cc3986a6 --- /dev/null +++ b/playbooks/dhall/diff.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - render-diff diff --git a/playbooks/dhall/prepare.yaml b/playbooks/dhall/prepare.yaml new file mode 100644 index 000000000..3a77d0510 --- /dev/null +++ b/playbooks/dhall/prepare.yaml @@ -0,0 +1,25 @@ +- hosts: all + tasks: + - name: Check for tools + command: type make git bzip2 + failed_when: false + register: _tools + + - name: Install tools + package: + name: + - git + - bzip2 + - make + become: true + when: _tools.rc != 0 + + - name: Check for dhall command + command: type -p dhall-to-json + failed_when: false + register: _dhall_tools + + - name: Install dhall + include_role: + name: ensure-dhall + when: _dhall_tools.rc != 0 diff --git a/roles/ensure-dhall/README.rst b/roles/ensure-dhall/README.rst new file mode 100644 index 000000000..08389597d --- /dev/null +++ b/roles/ensure-dhall/README.rst @@ -0,0 +1,10 @@ +Ensure dhall is installed + +Installs the specified version of the haskell implementation. + +**Role Variables** + +.. zuul:rolevar:: dhall_version + :default: 1.31.1 + + The dhall version. diff --git a/roles/ensure-dhall/defaults/main.yaml b/roles/ensure-dhall/defaults/main.yaml new file mode 100644 index 000000000..a0c944328 --- /dev/null +++ b/roles/ensure-dhall/defaults/main.yaml @@ -0,0 +1,9 @@ +dhall_version: 1.31.1 +dhall_versions: + "1.31.1": + - url: https://github.com/dhall-lang/dhall-haskell/releases/download/1.31.1/dhall-1.31.1-x86_64-linux.tar.bz2 + checksum: sha256:f8312727bbd4af74d183efce2e22f7b7807246a600fcc85600945f4790e4294a + - url: https://github.com/dhall-lang/dhall-haskell/releases/download/1.31.1/dhall-json-1.6.3-x86_64-linux.tar.bz2 + checksum: sha256:7ec6b54f70f077d8876bb40633905f2a0a2c69e83a3ce0bd3d9f7577e1210489 + - url: https://github.com/dhall-lang/dhall-haskell/releases/download/1.31.1/dhall-yaml-1.0.3-x86_64-linux.tar.bz2 + checksum: sha256:04d95773f9e96340a29f2857a55fb718735b83596abacf1d7b9fa3f6004067cb diff --git a/roles/ensure-dhall/tasks/main.yaml b/roles/ensure-dhall/tasks/main.yaml new file mode 100644 index 000000000..07f4b1a5b --- /dev/null +++ b/roles/ensure-dhall/tasks/main.yaml @@ -0,0 +1,33 @@ +- name: Create temp directory + tempfile: + state: directory + register: dhall_archive_tempdir + +- name: Check requested version + fail: + msg: | + Unknown dhall version: {{ dhall_version }}. + It needs to be defined in {{ opendev_url }}/roles/ensure-dhall/defaults/main.yaml + when: dhall_versions[dhall_version] is not defined + vars: + opendev_url: https://opendev.org/zuul/zuul-jobs/src/branch/master/ + +- name: Download tools + get_url: + url: "{{ zj_dhall_tool_item.url }}" + dest: "{{ dhall_archive_tempdir.path }}/{{ zj_dhall_tool_item.url | basename }}" + checksum: "{{ zj_dhall_tool_item.checksum }}" + loop: "{{ dhall_versions[dhall_version] }}" + loop_control: + loop_var: zj_dhall_tool_item + +- name: Unpack tools + command: "tar -xf {{ dhall_archive_tempdir.path }}/{{ zj_dhall_tool_item.url | basename }} --strip-components=2 -j --mode='a+x' -C /usr/local/bin" + become: yes + # ANSIBLE0006: Skip linting since it triggers on the "tar" command, + # but we prefer the command above + tags: + - skip_ansible_lint + loop: "{{ dhall_versions[dhall_version] }}" + loop_control: + loop_var: zj_dhall_tool_item diff --git a/roles/render-diff/README.rst b/roles/render-diff/README.rst new file mode 100644 index 000000000..f906c6e33 --- /dev/null +++ b/roles/render-diff/README.rst @@ -0,0 +1,15 @@ +Run render command and ensure there are no resulting differences. + +Ensure that generated configuration files are idempotent. + +**Role Variables** + +.. zuul:jobvar:: render_command + :default: make + + The command that render the configuration files. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to run the render command in. diff --git a/roles/render-diff/defaults/main.yaml b/roles/render-diff/defaults/main.yaml new file mode 100644 index 000000000..a08ebdc9c --- /dev/null +++ b/roles/render-diff/defaults/main.yaml @@ -0,0 +1,4 @@ +render_command: make + +zuul_work_dir: "{{ zuul.project.src_dir }}" +diff_ignore_errors: false diff --git a/roles/render-diff/tasks/main.yaml b/roles/render-diff/tasks/main.yaml new file mode 100644 index 000000000..06541265c --- /dev/null +++ b/roles/render-diff/tasks/main.yaml @@ -0,0 +1,38 @@ +- name: "Run render command: {{ render_command }}" + command: "{{ render_command }}" + args: + chdir: "{{ zuul_work_dir }}" + +- name: Check for diff + command: git diff + args: + chdir: "{{ zuul_work_dir }}" + register: render_diff + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but diff is not supported by ansible git module. + tags: + - skip_ansible_lint + +- name: Abort on diff + when: + - render_diff.stdout + - not diff_ignore_errors + failed_when: true + debug: + msg: | + The repository content is not consistent. + Please run `{{ render_command }}` before `git commit` + + {% if render_diff.stdout | regex_search('^-') %} + You commited change in a configuration file without using the equivalent source file. + Please update the source file first and run {{ render_command }}. + + {% else %} + You commited change in a file without running make. + Please run `{{ render_command }}` before `git commit`. + + {% endif %} + + The difference is: + + {{ git_diff.stdout }} diff --git a/test-playbooks/dhall/setup-project.yaml b/test-playbooks/dhall/setup-project.yaml new file mode 100644 index 000000000..bf2500018 --- /dev/null +++ b/test-playbooks/dhall/setup-project.yaml @@ -0,0 +1,23 @@ +- hosts: all + tasks: + - name: Setup files + copy: + content: "{{ item.content }}" + dest: "{{ zuul.project.src_dir }}/{{ item.dest }}" + loop: + - content: "all:\n\tdhall-to-yaml < test.dhall > test.yaml" + dest: Makefile + - content: "21 + 21" + dest: test.dhall + - content: "42\n" + dest: test.yaml + + - name: Commit changes + shell: | + if ! test -f ~/.gitconfig && ! test -d ~/.config/git ; then + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + fi + git add Makefile test.dhall test.yaml && git commit -m "test content" + args: + chdir: "{{ zuul.project.src_dir }}" diff --git a/test-playbooks/dhall/test-dhall-diff.yaml b/test-playbooks/dhall/test-dhall-diff.yaml new file mode 100644 index 000000000..72c013db3 --- /dev/null +++ b/test-playbooks/dhall/test-dhall-diff.yaml @@ -0,0 +1,31 @@ +- hosts: all + tasks: + - name: Success dhall-diff + include_role: + name: render-diff + + - name: Ensure dhall-diff succeeded + assert: + that: + - render_diff.stdout == '' + + - name: Introduce a difference + copy: + content: "44" + dest: "{{ zuul.project.src_dir }}/test.yaml" + + - name: Commit the difference + command: git commit -a -m "test update" + args: + chdir: "{{ zuul.project.src_dir }}" + + - name: Failed dhall-diff + include_role: + name: render-diff + vars: + diff_ignore_errors: yes + + - name: Ensure dhall-diff failed + assert: + that: + - render_diff.stdout != '' diff --git a/zuul-tests.d/dhall.yaml b/zuul-tests.d/dhall.yaml new file mode 100644 index 000000000..64b449d54 --- /dev/null +++ b/zuul-tests.d/dhall.yaml @@ -0,0 +1,17 @@ +- job: + name: zuul-jobs-test-dhall-diff + description: Test the dhall-diff job and roles + parent: dhall-diff + files: + - playbooks/dhall/.* + - roles/ensure-dhall/.* + - roles/render-diff/.* + pre-run: test-playbooks/dhall/setup-project.yaml + run: test-playbooks/dhall/test-dhall-diff.yaml + +- project: + check: + jobs: &id001 + - zuul-jobs-test-dhall-diff + gate: + jobs: *id001 diff --git a/zuul.d/general-jobs.yaml b/zuul.d/general-jobs.yaml index b29e78af9..46e60b0ef 100644 --- a/zuul.d/general-jobs.yaml +++ b/zuul.d/general-jobs.yaml @@ -93,3 +93,24 @@ Override the default searching above with explicit domain/path references (see validate-zone-db role) run: playbooks/validate-zone-db/run.yaml + +- job: + name: dhall-diff + description: | + Ensure that generated configuration files are idempotent. + + This job runs a render command and check that no files are + modified. + + .. zuul:jobvar:: dhall_render_command + :default: make + + The command that render the configuration files. + + .. zuul:jobvar:: dhall_version + :default: 1.31.1 + + The dhall version. + + pre-run: playbooks/dhall/prepare.yaml + run: playbooks/dhall/diff.yaml