From fe58fb994711349264a809b2a8db79c48f865091 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Sat, 27 Jul 2024 13:03:05 +0200 Subject: [PATCH] Prepare git patch and archive for rust package Apply the generated sources on top of the target repository and create patch file. In addition to that save complete content as job artifacts. post job is pushing changes to the upstream repository (using public SSH key of repository set by Zuul and added as a deployment key). PR is not created, since that requires additionally separate API key. Change-Id: I090b87e05ede95f8542a427207bea711e5cc5df4 --- metadata/object-store_metadata.yaml | 116 ++++++++++++++-------------- playbooks/rust/all.yaml | 70 ++++++++++++++++- playbooks/rust/post.yaml | 72 +++++++++++++++++ playbooks/rust/propose-github.yaml | 61 +++++++++++++++ roles/codegenerator/tasks/main.yaml | 17 +++- zuul.d/project.yaml | 3 + zuul.d/rust.yaml | 39 +++++++++- 7 files changed, 315 insertions(+), 63 deletions(-) create mode 100644 playbooks/rust/post.yaml create mode 100644 playbooks/rust/propose-github.yaml diff --git a/metadata/object-store_metadata.yaml b/metadata/object-store_metadata.yaml index 86ff53f..6d0bcbd 100644 --- a/metadata/object-store_metadata.yaml +++ b/metadata/object-store_metadata.yaml @@ -9,43 +9,47 @@ resources: targets: rust-sdk: module_name: head - rust-cli: - module_name: head - sdk_mod_name: head - cli_full_command: account show + #rust-cli: + # module_name: head + # sdk_mod_name: head + # cli_full_command: account show get: operation_id: account.get operation_type: get targets: rust-sdk: module_name: get - rust-cli: - module_name: get - sdk_mod_name: get - cli_full_command: container list + #rust-cli: + # module_name: get + # sdk_mod_name: get + # cli_full_command: container list update: operation_id: account.post operation_type: set targets: rust-sdk: module_name: set - rust-cli: - module_name: set - sdk_mod_name: set - cli_full_command: account set + #rust-cli: + # module_name: set + # sdk_mod_name: set + # cli_full_command: account set delete: operation_id: account.delete operation_type: delete targets: rust-sdk: module_name: delete - rust-cli: - module_name: delete - sdk_mod_name: delete - cli_full_command: account delete + #rust-cli: + # module_name: delete + # sdk_mod_name: delete + # cli_full_command: account delete object-store.container: spec_file: wrk/openapi_specs/object-store/v1.yaml api_version: v1 + extensions: + rust-sdk: + additional_modules: + - prune operations: head: operation_id: container.head @@ -53,50 +57,50 @@ resources: targets: rust-sdk: module_name: head - rust-cli: - module_name: head - sdk_mod_name: head - cli_full_command: container show + #rust-cli: + # module_name: head + # sdk_mod_name: head + # cli_full_command: container show get: operation_id: container.get operation_type: get targets: rust-sdk: module_name: get - rust-cli: - module_name: get - sdk_mod_name: get - cli_full_command: object list + #rust-cli: + # module_name: get + # sdk_mod_name: get + # cli_full_command: object list create: operation_id: container.put operation_type: create targets: rust-sdk: module_name: create - rust-cli: - module_name: create - sdk_mod_name: create - cli_full_command: container create + #rust-cli: + # module_name: create + # sdk_mod_name: create + # cli_full_command: container create update: operation_id: container.post operation_type: set targets: rust-sdk: module_name: set - rust-cli: - module_name: set - sdk_mod_name: set - cli_full_command: container set + #rust-cli: + # module_name: set + # sdk_mod_name: set + # cli_full_command: container set delete: operation_id: container.delete operation_type: delete targets: rust-sdk: module_name: delete - rust-cli: - module_name: delete - sdk_mod_name: delete - cli_full_command: container delete + #rust-cli: + # module_name: delete + # sdk_mod_name: delete + # cli_full_command: container delete object-store.object: spec_file: wrk/openapi_specs/object-store/v1.yaml api_version: v1 @@ -107,47 +111,47 @@ resources: targets: rust-sdk: module_name: head - rust-cli: - module_name: head - sdk_mod_name: head - cli_full_command: object show + #rust-cli: + # module_name: head + # sdk_mod_name: head + # cli_full_command: object show get: operation_id: object.get operation_type: download targets: rust-sdk: module_name: get - rust-cli: - module_name: get - sdk_mod_name: get - cli_full_command: object download + #rust-cli: + # module_name: get + # sdk_mod_name: get + # cli_full_command: object download put: operation_id: object.put operation_type: upload targets: rust-sdk: module_name: put - rust-cli: - module_name: put - sdk_mod_name: put - cli_full_command: object upload + #rust-cli: + # module_name: put + # sdk_mod_name: put + # cli_full_command: object upload update: operation_id: object.post operation_type: set targets: rust-sdk: module_name: set - rust-cli: - module_name: set - sdk_mod_name: set - cli_full_command: object set + #rust-cli: + # module_name: set + # sdk_mod_name: set + # cli_full_command: object set delete: operation_id: object.delete operation_type: delete targets: rust-sdk: module_name: delete - rust-cli: - module_name: delete - sdk_mod_name: delete - cli_full_command: object delete + #rust-cli: + # module_name: delete + # sdk_mod_name: delete + # cli_full_command: object delete diff --git a/playbooks/rust/all.yaml b/playbooks/rust/all.yaml index b53ba5b..b20a683 100644 --- a/playbooks/rust/all.yaml +++ b/playbooks/rust/all.yaml @@ -3,9 +3,9 @@ roles: - "ensure-rust" tasks: - - name: Generate Rust code + - name: "Generate Rust code" ansible.builtin.include_role: - name: codegenerator + name: "codegenerator" vars: codegenerator_target: "{{ zj_item.1 }}" codegenerator_metadata: "{{ zj_item.0.metadata }}" @@ -13,3 +13,69 @@ loop: "{{ codegenerator_service_metadata_target_map | subelements('targets') }}" loop_control: loop_var: zj_item + + - name: "Checkout target repository" + ansible.builtin.git: + repo: "{{ rust_sdk_git_repo }}" + dest: "{{ rust_project_dir }}" + + - name: "Pre-Compile current code to ensure it builds" + ansible.builtin.command: + cmd: "cargo build" + chdir: "{{ rust_project_dir }}" + + - name: "Overwrite generated files" + ansible.builtin.copy: + src: "{{ codegenerator_base_dir }}/wrk/rust/" + dest: "{{ rust_project_dir }}" + remote_src: True + + - name: "Optimize generated code with clippy" + ansible.builtin.command: + cmd: "cargo clippy --fix --lib --tests --allow-dirty" + chdir: "{{ rust_project_dir }}" + + - name: "Compile new code" + ansible.builtin.command: + cmd: "cargo build" + chdir: "{{ rust_project_dir }}" + + - name: "Checkout new branch" + ansible.builtin.command: + cmd: "git checkout -b codegenerator_{{ zuul.change }}" + chdir: "{{ rust_project_dir }}" + + - name: "Configure git username" + ansible.builtin.command: "git config --global user.name 'OpenStack codegenerator'" + + - name: "Configure git email" + ansible.builtin.command: "git config --global user.email 16461884+gtema@users.noreply.github.com" + + - name: "Stage files for commit" + ansible.builtin.command: + cmd: "git add ." + chdir: "{{ rust_project_dir }}" + + - name: "Check staged files" + ansible.builtin.command: + cmd: "git diff --staged" + chdir: "{{ rust_project_dir }}" + register: "staged_changes" + + - name: "Commit changes" + ansible.builtin.command: + cmd: "git commit -m 'feat: New generated content' -m '{{ zuul.change_message }}' -m 'Changes are triggered by {{ zuul.change_url }}'" + chdir: "{{ rust_project_dir }}" + register: "commit" + when: + # Only commit when there is anything to commit + - "staged_changes.stdout | length > 0" + + - name: "Format patch" + ansible.builtin.command: + cmd: "git format-patch -1 --output {{ patch_path }}" + chdir: "{{ rust_project_dir }}" + when: + # Only prepare when there is anything to commit + - "commit is defined" + - "commit.changed" diff --git a/playbooks/rust/post.yaml b/playbooks/rust/post.yaml new file mode 100644 index 0000000..10a2ef4 --- /dev/null +++ b/playbooks/rust/post.yaml @@ -0,0 +1,72 @@ +--- +- hosts: all + vars: + archive_path: "{{ ansible_user_dir }}/archive.tar.gz" + tasks: + - name: "Ensure local output dirs" + delegate_to: localhost + ansible.builtin.file: + path: "{{ zj_output_dir }}" + state: directory + mode: 0755 + with_items: + - "{{ zuul.executor.work_root }}/artifacts" + loop_control: + loop_var: zj_output_dir + + - name: "Archive generated code" + community.general.archive: + path: "{{ codegenerator_base_dir }}/wrk/rust" + dest: "{{ archive_path }}" + + - name: "Collect archived generated code" + ansible.posix.synchronize: + dest: "{{ zj_output.dest }}" + mode: pull + src: "{{ zj_output.src }}" + verify_host: true + owner: false + group: false + loop: + - dest: "{{ zuul.executor.work_root }}/artifacts/" + src: "{{ archive_path }}" + loop_control: + loop_var: zj_output + + - name: "Return source code artifact to Zuul" + zuul_return: + data: + zuul: + artifacts: + - name: "Code changes" + url: "artifacts/archive.tar.gz" + metadata: + type: "content" + + - name: Find generated patch file + ansible.builtin.stat: + path: "{{ patch_path }}" + register: patch_stat + + - name: "Collect patch file" + ansible.posix.synchronize: + dest: "{{ zuul.executor.work_root }}/artifacts/" + mode: pull + src: "{{ patch_path }}" + verify_host: true + owner: false + group: false + when: + - "patch_stat.stat.exists" + + - name: "Return patch artifact to Zuul" + zuul_return: + data: + zuul: + artifacts: + - name: "Git patch" + url: "artifacts/{{ zuul.change }}.patch" + metadata: + type: "rust_patch" + when: + - "patch_stat.stat.exists" diff --git a/playbooks/rust/propose-github.yaml b/playbooks/rust/propose-github.yaml new file mode 100644 index 0000000..0a4bf9c --- /dev/null +++ b/playbooks/rust/propose-github.yaml @@ -0,0 +1,61 @@ +--- +- name: "Propose generated code changes upstream" + hosts: localhost + vars: + patch_file: "{{ zuul.executor.work_root }}/{{ zuul.change }}.patch" + branch_name: "codegenerator_{{ zuul.change }}" + tasks: + + - name: "Check execution context" + when: "zuul.branch is not defined" + fail: + msg: "This playbook must be run in a branch-based pipeline (e.g., 'promote')." + + - name: "Download docs archive" + ansible.builtin.include_role: + name: download-artifact + vars: + # download_artifact_job: provided by zuul job + download_artifact_api: "https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}" + download_artifact_type: + - rust_patch + download_artifact_pipeline: gate + + - name: "Check git patch presense" + ansible.builtin.stat: + path: "{{ patch_file }}" + register: "git_patch_stat" + + - name: "Process patch" + when: + - "git_patch_stat.stat.exists" + block: + + - name: "Checkout target repository" + ansible.builtin.git: + repo: "{{ rust_sdk_git_repo }}" + dest: "{{ rust_project_dir }}" + + - name: "Checkout new branch" + ansible.builtin.command: "git checkout -b {{ branch_name }}" + args: + chdir: "{{ ansible_user_dir }}/openstack" + + - name: "Try to apply git patch" + ansible.builtin.command: "git apply --reject --ignore-space-change {{ git_patch_stat.stat.path }}" + args: + chdir: "{{ ansible_user_dir }}/openstack" + register: "patch_applied" + failed_when: false + + - name: "Push changes" + ansible.builtin.command: "git push" + args: + chdir: "{{ ansible_user_dir }}/openstack" + when: "patch_applied.changed" + register: "change_pushed" + + - name: "Inform how to open PR" + ansible.builtin.debug: + msg: "Please follow the link https://github.com/gtema/openstack/pull/new/{{ branch_name }} to create new pull request" + when: "change_pushed.changed" diff --git a/roles/codegenerator/tasks/main.yaml b/roles/codegenerator/tasks/main.yaml index 0541b57..b851bf0 100644 --- a/roles/codegenerator/tasks/main.yaml +++ b/roles/codegenerator/tasks/main.yaml @@ -1,5 +1,18 @@ --- -- name: Invoke openstack-codegenerator +- name: "Read service metadata" + ansible.builtin.slurp: + src: "{{ codegenerator_base_dir }}/{{ codegenerator_metadata }}" + register: "metadata_content" + +- ansible.builtin.set_fact: + metadata: "{{ metadata_content.content | b64decode | from_yaml }}" + +- name: "Check presense of openapi file (first one)" + ansible.builtin.stat: + path: "{{ codegenerator_base_dir }}/{{ metadata.resources | dict2items | map(attribute='value') | map(attribute='spec_file') | first }}" + register: "openapi_stat" + +- name: "Invoke openstack-codegenerator" args: chdir: "{{ codegenerator_base_dir }}" executable: "/bin/bash" @@ -14,3 +27,5 @@ {% if codegenerator_service_type is defined %} --service {{ codegenerator_service_type }} {% endif %} + when: + - "openapi_stat.stat.exists" diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 26ff430..849dc9d 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -32,3 +32,6 @@ - codegenerator-openapi-shared-file-system-tips-with-api-ref - codegenerator-tox-publish-openapi-specs - codegenerator-rust-all + post: + jobs: + - codegenerator-propose-rust-openstack-change diff --git a/zuul.d/rust.yaml b/zuul.d/rust.yaml index 1a79b66..114da44 100644 --- a/zuul.d/rust.yaml +++ b/zuul.d/rust.yaml @@ -8,6 +8,8 @@ timeout: 1800 pre-run: - playbooks/codegenerator/pre.yaml + post-run: + - playbooks/rust/post.yaml vars: codegenerator_service_metadata_target_map: - service: "block-storage" @@ -19,9 +21,11 @@ - service: "identity" metadata: "metadata/identity_metadata.yaml" targets: ["rust-sdk", "rust-cli"] - - service: "image" - metadata: "metadata/image_metadata.yaml" - targets: ["rust-sdk", "rust-cli"] + # https://review.opendev.org/c/openstack/glance/+/882498 screwed us. + # Disable build for glance until we find way to deal with that + # - service: "image" + # metadata: "metadata/image_metadata.yaml" + # targets: ["rust-sdk", "rust-cli"] - service: "load-balancer" metadata: "metadata/load-balancer_metadata.yaml" targets: ["rust-sdk", "rust-cli"] @@ -38,6 +42,8 @@ - job: name: codegenerator-rust-all parent: codegenerator-rust-base + # It takes a while to compile the project + timeout: 3600 description: | Generate Rust SDK/CLI dependencies: @@ -62,5 +68,30 @@ pre-run: playbooks/openapi/fetch.yaml run: playbooks/rust/all.yaml vars: - codegenerator_base_dir: "{{ zuul.project.src_dir }}" + codegenerator_base_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" openapi_dest: "{{ codegenerator_base_dir }}/wrk/openapi_specs" + patch_path: "{{ ansible_user_dir }}/{{ zuul.change }}.patch" + rust_sdk_git_repo: "https://github.com/gtema/openstack" + rust_project_dir: "{{ ansible_user_dir }}/openstack" + +- job: + name: codegenerator-propose-rust-openstack-change + description: | + Propose changes to the Rust OpenStack project with the code newly + generated in this change. + + This job is applying the git patch created by the + `codegenerator-rust-all` job. It accesses target repository using SSH + and a public key of the project (`curl + https://zuul.opendev.org/api/tenant/openstack/project-ssh-key/openstack/codegenerator.pub`) + being added as a deploy key in GitHub repository with read/write access. + Pull Request itself is not opened, since it requires additionally API + token what would require hardcoding secret (and GitHub does everything + possible not to have long lasting tokens). + post-review: true + run: playbooks/rust/propose-github.yaml + nodeset: + nodes: [] + vars: + rust_sdk_git_repo: "git@github.com:gtema/openstack.git" + rust_project_dir: "{{ ansible_user_dir }}/openstack"