openstack/codegenerator
Code Issues Proposed changes

Prepare git patch and archive for rust package

Browse Source 
Apply the generated sources on top of the target repository and create
patch file. In addition to that save complete content as job artifacts.

post job is pushing changes to the upstream repository (using public SSH
key of repository set by Zuul and added as a deployment key). PR is not
created, since that requires additionally separate API key.

Change-Id: I090b87e05ede95f8542a427207bea711e5cc5df4
This commit is contained in:
Artem Goncharov 2024-07-27 13:03:05 +02:00
parent 600d7aca5a
commit fe58fb9947
7 changed files with 315 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
metadata/object-store_metadata.yaml
View File

@ -9,43 +9,47 @@ resources:
targets:
rust-sdk:
module_name: head
rust-cli:
module_name: head
sdk_mod_name: head
cli_full_command: account show
#rust-cli:
# module_name: head
# sdk_mod_name: head
# cli_full_command: account show
get:
operation_id: account.get
operation_type: get
targets:
rust-sdk:
module_name: get
rust-cli:
module_name: get
sdk_mod_name: get
cli_full_command: container list
#rust-cli:
# module_name: get
# sdk_mod_name: get
# cli_full_command: container list
update:
operation_id: account.post
operation_type: set
targets:
rust-sdk:
module_name: set
rust-cli:
module_name: set
sdk_mod_name: set
cli_full_command: account set
#rust-cli:
# module_name: set
# sdk_mod_name: set
# cli_full_command: account set
delete:
operation_id: account.delete
operation_type: delete
targets:
rust-sdk:
module_name: delete
rust-cli:
module_name: delete
sdk_mod_name: delete
cli_full_command: account delete
#rust-cli:
# module_name: delete
# sdk_mod_name: delete
# cli_full_command: account delete
object-store.container:
spec_file: wrk/openapi_specs/object-store/v1.yaml
api_version: v1
extensions:
rust-sdk:
additional_modules:
- prune
operations:
head:
operation_id: container.head
@ -53,50 +57,50 @@ resources:
targets:
rust-sdk:
module_name: head
rust-cli:
module_name: head
sdk_mod_name: head
cli_full_command: container show
#rust-cli:
# module_name: head
# sdk_mod_name: head
# cli_full_command: container show
get:
operation_id: container.get
operation_type: get
targets:
rust-sdk:
module_name: get
rust-cli:
module_name: get
sdk_mod_name: get
cli_full_command: object list
#rust-cli:
# module_name: get
# sdk_mod_name: get
# cli_full_command: object list
create:
operation_id: container.put
operation_type: create
targets:
rust-sdk:
module_name: create
rust-cli:
module_name: create
sdk_mod_name: create
cli_full_command: container create
#rust-cli:
# module_name: create
# sdk_mod_name: create
# cli_full_command: container create
update:
operation_id: container.post
operation_type: set
targets:
rust-sdk:
module_name: set
rust-cli:
module_name: set
sdk_mod_name: set
cli_full_command: container set
#rust-cli:
# module_name: set
# sdk_mod_name: set
# cli_full_command: container set
delete:
operation_id: container.delete
operation_type: delete
targets:
rust-sdk:
module_name: delete
rust-cli:
module_name: delete
sdk_mod_name: delete
cli_full_command: container delete
#rust-cli:
# module_name: delete
# sdk_mod_name: delete
# cli_full_command: container delete
object-store.object:
spec_file: wrk/openapi_specs/object-store/v1.yaml
api_version: v1
@ -107,47 +111,47 @@ resources:
targets:
rust-sdk:
module_name: head
rust-cli:
module_name: head
sdk_mod_name: head
cli_full_command: object show
#rust-cli:
# module_name: head
# sdk_mod_name: head
# cli_full_command: object show
get:
operation_id: object.get
operation_type: download
targets:
rust-sdk:
module_name: get
rust-cli:
module_name: get
sdk_mod_name: get
cli_full_command: object download
#rust-cli:
# module_name: get
# sdk_mod_name: get
# cli_full_command: object download
put:
operation_id: object.put
operation_type: upload
targets:
rust-sdk:
module_name: put
rust-cli:
module_name: put
sdk_mod_name: put
cli_full_command: object upload
#rust-cli:
# module_name: put
# sdk_mod_name: put
# cli_full_command: object upload
update:
operation_id: object.post
operation_type: set
targets:
rust-sdk:
module_name: set
rust-cli:
module_name: set
sdk_mod_name: set
cli_full_command: object set
#rust-cli:
# module_name: set
# sdk_mod_name: set
# cli_full_command: object set
delete:
operation_id: object.delete
operation_type: delete
targets:
rust-sdk:
module_name: delete
rust-cli:
module_name: delete
sdk_mod_name: delete
cli_full_command: object delete
#rust-cli:
# module_name: delete
# sdk_mod_name: delete
# cli_full_command: object delete

70
playbooks/rust/all.yaml
View File

@ -3,9 +3,9 @@
roles:
- "ensure-rust"
tasks:
- name: Generate Rust code
- name: "Generate Rust code"
ansible.builtin.include_role:
name: codegenerator
name: "codegenerator"
vars:
codegenerator_target: "{{ zj_item.1 }}"
codegenerator_metadata: "{{ zj_item.0.metadata }}"
@ -13,3 +13,69 @@
loop: "{{ codegenerator_service_metadata_target_map | subelements('targets') }}"
loop_control:
loop_var: zj_item
- name: "Checkout target repository"
ansible.builtin.git:
repo: "{{ rust_sdk_git_repo }}"
dest: "{{ rust_project_dir }}"
- name: "Pre-Compile current code to ensure it builds"
ansible.builtin.command:
cmd: "cargo build"
chdir: "{{ rust_project_dir }}"
- name: "Overwrite generated files"
ansible.builtin.copy:
src: "{{ codegenerator_base_dir }}/wrk/rust/"
dest: "{{ rust_project_dir }}"
remote_src: True
- name: "Optimize generated code with clippy"
ansible.builtin.command:
cmd: "cargo clippy --fix --lib --tests --allow-dirty"
chdir: "{{ rust_project_dir }}"
- name: "Compile new code"
ansible.builtin.command:
cmd: "cargo build"
chdir: "{{ rust_project_dir }}"
- name: "Checkout new branch"
ansible.builtin.command:
cmd: "git checkout -b codegenerator_{{ zuul.change }}"
chdir: "{{ rust_project_dir }}"
- name: "Configure git username"
ansible.builtin.command: "git config --global user.name 'OpenStack codegenerator'"
- name: "Configure git email"
ansible.builtin.command: "git config --global user.email 16461884+gtema@users.noreply.github.com"
- name: "Stage files for commit"
ansible.builtin.command:
cmd: "git add ."
chdir: "{{ rust_project_dir }}"
- name: "Check staged files"
ansible.builtin.command:
cmd: "git diff --staged"
chdir: "{{ rust_project_dir }}"
register: "staged_changes"
- name: "Commit changes"
ansible.builtin.command:
cmd: "git commit -m 'feat: New generated content' -m '{{ zuul.change_message }}' -m 'Changes are triggered by {{ zuul.change_url }}'"
chdir: "{{ rust_project_dir }}"
register: "commit"
when:
# Only commit when there is anything to commit
- "staged_changes.stdout | length > 0"
- name: "Format patch"
ansible.builtin.command:
cmd: "git format-patch -1 --output {{ patch_path }}"
chdir: "{{ rust_project_dir }}"
when:
# Only prepare when there is anything to commit
- "commit is defined"
- "commit.changed"

72
playbooks/rust/post.yaml Normal file
View File

@ -0,0 +1,72 @@
---
- hosts: all
vars:
archive_path: "{{ ansible_user_dir }}/archive.tar.gz"
tasks:
- name: "Ensure local output dirs"
delegate_to: localhost
ansible.builtin.file:
path: "{{ zj_output_dir }}"
state: directory
mode: 0755
with_items:
- "{{ zuul.executor.work_root }}/artifacts"
loop_control:
loop_var: zj_output_dir
- name: "Archive generated code"
community.general.archive:
path: "{{ codegenerator_base_dir }}/wrk/rust"
dest: "{{ archive_path }}"
- name: "Collect archived generated code"
ansible.posix.synchronize:
dest: "{{ zj_output.dest }}"
mode: pull
src: "{{ zj_output.src }}"
verify_host: true
owner: false
group: false
loop:
- dest: "{{ zuul.executor.work_root }}/artifacts/"
src: "{{ archive_path }}"
loop_control:
loop_var: zj_output
- name: "Return source code artifact to Zuul"
zuul_return:
data:
zuul:
artifacts:
- name: "Code changes"
url: "artifacts/archive.tar.gz"
metadata:
type: "content"
- name: Find generated patch file
ansible.builtin.stat:
path: "{{ patch_path }}"
register: patch_stat
- name: "Collect patch file"
ansible.posix.synchronize:
dest: "{{ zuul.executor.work_root }}/artifacts/"
mode: pull
src: "{{ patch_path }}"
verify_host: true
owner: false
group: false
when:
- "patch_stat.stat.exists"
- name: "Return patch artifact to Zuul"
zuul_return:
data:
zuul:
artifacts:
- name: "Git patch"
url: "artifacts/{{ zuul.change }}.patch"
metadata:
type: "rust_patch"
when:
- "patch_stat.stat.exists"

61
playbooks/rust/propose-github.yaml Normal file
View File

@ -0,0 +1,61 @@
---
- name: "Propose generated code changes upstream"
hosts: localhost
vars:
patch_file: "{{ zuul.executor.work_root }}/{{ zuul.change }}.patch"
branch_name: "codegenerator_{{ zuul.change }}"
tasks:
- name: "Check execution context"
when: "zuul.branch is not defined"
fail:
msg: "This playbook must be run in a branch-based pipeline (e.g., 'promote')."
- name: "Download docs archive"
ansible.builtin.include_role:
name: download-artifact
vars:
# download_artifact_job: provided by zuul job
download_artifact_api: "https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}"
download_artifact_type:
- rust_patch
download_artifact_pipeline: gate
- name: "Check git patch presense"
ansible.builtin.stat:
path: "{{ patch_file }}"
register: "git_patch_stat"
- name: "Process patch"
when:
- "git_patch_stat.stat.exists"
block:
- name: "Checkout target repository"
ansible.builtin.git:
repo: "{{ rust_sdk_git_repo }}"
dest: "{{ rust_project_dir }}"
- name: "Checkout new branch"
ansible.builtin.command: "git checkout -b {{ branch_name }}"
args:
chdir: "{{ ansible_user_dir }}/openstack"
- name: "Try to apply git patch"
ansible.builtin.command: "git apply --reject --ignore-space-change {{ git_patch_stat.stat.path }}"
args:
chdir: "{{ ansible_user_dir }}/openstack"
register: "patch_applied"
failed_when: false
- name: "Push changes"
ansible.builtin.command: "git push"
args:
chdir: "{{ ansible_user_dir }}/openstack"
when: "patch_applied.changed"
register: "change_pushed"
- name: "Inform how to open PR"
ansible.builtin.debug:
msg: "Please follow the link https://github.com/gtema/openstack/pull/new/{{ branch_name }} to create new pull request"
when: "change_pushed.changed"

17
roles/codegenerator/tasks/main.yaml
View File

@ -1,5 +1,18 @@
---
- name: Invoke openstack-codegenerator
- name: "Read service metadata"
ansible.builtin.slurp:
src: "{{ codegenerator_base_dir }}/{{ codegenerator_metadata }}"
register: "metadata_content"
- ansible.builtin.set_fact:
metadata: "{{ metadata_content.content | b64decode | from_yaml }}"
- name: "Check presense of openapi file (first one)"
ansible.builtin.stat:
path: "{{ codegenerator_base_dir }}/{{ metadata.resources | dict2items | map(attribute='value') | map(attribute='spec_file') | first }}"
register: "openapi_stat"
- name: "Invoke openstack-codegenerator"
args:
chdir: "{{ codegenerator_base_dir }}"
executable: "/bin/bash"
@ -14,3 +27,5 @@
{% if codegenerator_service_type is defined %}
--service {{ codegenerator_service_type }}
{% endif %}
when:
- "openapi_stat.stat.exists"

3
zuul.d/project.yaml
View File

@ -32,3 +32,6 @@
- codegenerator-openapi-shared-file-system-tips-with-api-ref
- codegenerator-tox-publish-openapi-specs
- codegenerator-rust-all
post:
jobs:
- codegenerator-propose-rust-openstack-change

39
zuul.d/rust.yaml
View File

@ -8,6 +8,8 @@
timeout: 1800
pre-run:
- playbooks/codegenerator/pre.yaml
post-run:
- playbooks/rust/post.yaml
vars:
codegenerator_service_metadata_target_map:
- service: "block-storage"
@ -19,9 +21,11 @@
- service: "identity"
metadata: "metadata/identity_metadata.yaml"
targets: ["rust-sdk", "rust-cli"]
- service: "image"
metadata: "metadata/image_metadata.yaml"
targets: ["rust-sdk", "rust-cli"]
# https://review.opendev.org/c/openstack/glance/+/882498 screwed us.
# Disable build for glance until we find way to deal with that
# - service: "image"
# metadata: "metadata/image_metadata.yaml"
# targets: ["rust-sdk", "rust-cli"]
- service: "load-balancer"
metadata: "metadata/load-balancer_metadata.yaml"
targets: ["rust-sdk", "rust-cli"]
@ -38,6 +42,8 @@
- job:
name: codegenerator-rust-all
parent: codegenerator-rust-base
# It takes a while to compile the project
timeout: 3600
description: |
Generate Rust SDK/CLI
dependencies:
@ -62,5 +68,30 @@
pre-run: playbooks/openapi/fetch.yaml
run: playbooks/rust/all.yaml
vars:
codegenerator_base_dir: "{{ zuul.project.src_dir }}"
codegenerator_base_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
openapi_dest: "{{ codegenerator_base_dir }}/wrk/openapi_specs"
patch_path: "{{ ansible_user_dir }}/{{ zuul.change }}.patch"
rust_sdk_git_repo: "https://github.com/gtema/openstack"
rust_project_dir: "{{ ansible_user_dir }}/openstack"
- job:
name: codegenerator-propose-rust-openstack-change
description: |
Propose changes to the Rust OpenStack project with the code newly
generated in this change.
This job is applying the git patch created by the
`codegenerator-rust-all` job. It accesses target repository using SSH
and a public key of the project (`curl
https://zuul.opendev.org/api/tenant/openstack/project-ssh-key/openstack/codegenerator.pub`)
being added as a deploy key in GitHub repository with read/write access.
Pull Request itself is not opened, since it requires additionally API
token what would require hardcoding secret (and GitHub does everything
possible not to have long lasting tokens).
post-review: true
run: playbooks/rust/propose-github.yaml
nodeset:
nodes: []
vars:
rust_sdk_git_repo: "git@github.com:gtema/openstack.git"
rust_project_dir: "{{ ansible_user_dir }}/openstack"