diff --git a/metadata/object-store_metadata.yaml b/metadata/object-store_metadata.yaml index 86ff53f..6d0bcbd 100644 --- a/metadata/object-store_metadata.yaml +++ b/metadata/object-store_metadata.yaml @@ -9,43 +9,47 @@ resources: targets: rust-sdk: module_name: head - rust-cli: - module_name: head - sdk_mod_name: head - cli_full_command: account show + #rust-cli: + # module_name: head + # sdk_mod_name: head + # cli_full_command: account show get: operation_id: account.get operation_type: get targets: rust-sdk: module_name: get - rust-cli: - module_name: get - sdk_mod_name: get - cli_full_command: container list + #rust-cli: + # module_name: get + # sdk_mod_name: get + # cli_full_command: container list update: operation_id: account.post operation_type: set targets: rust-sdk: module_name: set - rust-cli: - module_name: set - sdk_mod_name: set - cli_full_command: account set + #rust-cli: + # module_name: set + # sdk_mod_name: set + # cli_full_command: account set delete: operation_id: account.delete operation_type: delete targets: rust-sdk: module_name: delete - rust-cli: - module_name: delete - sdk_mod_name: delete - cli_full_command: account delete + #rust-cli: + # module_name: delete + # sdk_mod_name: delete + # cli_full_command: account delete object-store.container: spec_file: wrk/openapi_specs/object-store/v1.yaml api_version: v1 + extensions: + rust-sdk: + additional_modules: + - prune operations: head: operation_id: container.head @@ -53,50 +57,50 @@ resources: targets: rust-sdk: module_name: head - rust-cli: - module_name: head - sdk_mod_name: head - cli_full_command: container show + #rust-cli: + # module_name: head + # sdk_mod_name: head + # cli_full_command: container show get: operation_id: container.get operation_type: get targets: rust-sdk: module_name: get - rust-cli: - module_name: get - sdk_mod_name: get - cli_full_command: object list + #rust-cli: + # module_name: get + # sdk_mod_name: get + # cli_full_command: object list create: operation_id: container.put operation_type: create targets: rust-sdk: module_name: create - rust-cli: - module_name: create - sdk_mod_name: create - cli_full_command: container create + #rust-cli: + # module_name: create + # sdk_mod_name: create + # cli_full_command: container create update: operation_id: container.post operation_type: set targets: rust-sdk: module_name: set - rust-cli: - module_name: set - sdk_mod_name: set - cli_full_command: container set + #rust-cli: + # module_name: set + # sdk_mod_name: set + # cli_full_command: container set delete: operation_id: container.delete operation_type: delete targets: rust-sdk: module_name: delete - rust-cli: - module_name: delete - sdk_mod_name: delete - cli_full_command: container delete + #rust-cli: + # module_name: delete + # sdk_mod_name: delete + # cli_full_command: container delete object-store.object: spec_file: wrk/openapi_specs/object-store/v1.yaml api_version: v1 @@ -107,47 +111,47 @@ resources: targets: rust-sdk: module_name: head - rust-cli: - module_name: head - sdk_mod_name: head - cli_full_command: object show + #rust-cli: + # module_name: head + # sdk_mod_name: head + # cli_full_command: object show get: operation_id: object.get operation_type: download targets: rust-sdk: module_name: get - rust-cli: - module_name: get - sdk_mod_name: get - cli_full_command: object download + #rust-cli: + # module_name: get + # sdk_mod_name: get + # cli_full_command: object download put: operation_id: object.put operation_type: upload targets: rust-sdk: module_name: put - rust-cli: - module_name: put - sdk_mod_name: put - cli_full_command: object upload + #rust-cli: + # module_name: put + # sdk_mod_name: put + # cli_full_command: object upload update: operation_id: object.post operation_type: set targets: rust-sdk: module_name: set - rust-cli: - module_name: set - sdk_mod_name: set - cli_full_command: object set + #rust-cli: + # module_name: set + # sdk_mod_name: set + # cli_full_command: object set delete: operation_id: object.delete operation_type: delete targets: rust-sdk: module_name: delete - rust-cli: - module_name: delete - sdk_mod_name: delete - cli_full_command: object delete + #rust-cli: + # module_name: delete + # sdk_mod_name: delete + # cli_full_command: object delete diff --git a/playbooks/rust/all.yaml b/playbooks/rust/all.yaml index b53ba5b..b20a683 100644 --- a/playbooks/rust/all.yaml +++ b/playbooks/rust/all.yaml @@ -3,9 +3,9 @@ roles: - "ensure-rust" tasks: - - name: Generate Rust code + - name: "Generate Rust code" ansible.builtin.include_role: - name: codegenerator + name: "codegenerator" vars: codegenerator_target: "{{ zj_item.1 }}" codegenerator_metadata: "{{ zj_item.0.metadata }}" @@ -13,3 +13,69 @@ loop: "{{ codegenerator_service_metadata_target_map | subelements('targets') }}" loop_control: loop_var: zj_item + + - name: "Checkout target repository" + ansible.builtin.git: + repo: "{{ rust_sdk_git_repo }}" + dest: "{{ rust_project_dir }}" + + - name: "Pre-Compile current code to ensure it builds" + ansible.builtin.command: + cmd: "cargo build" + chdir: "{{ rust_project_dir }}" + + - name: "Overwrite generated files" + ansible.builtin.copy: + src: "{{ codegenerator_base_dir }}/wrk/rust/" + dest: "{{ rust_project_dir }}" + remote_src: True + + - name: "Optimize generated code with clippy" + ansible.builtin.command: + cmd: "cargo clippy --fix --lib --tests --allow-dirty" + chdir: "{{ rust_project_dir }}" + + - name: "Compile new code" + ansible.builtin.command: + cmd: "cargo build" + chdir: "{{ rust_project_dir }}" + + - name: "Checkout new branch" + ansible.builtin.command: + cmd: "git checkout -b codegenerator_{{ zuul.change }}" + chdir: "{{ rust_project_dir }}" + + - name: "Configure git username" + ansible.builtin.command: "git config --global user.name 'OpenStack codegenerator'" + + - name: "Configure git email" + ansible.builtin.command: "git config --global user.email 16461884+gtema@users.noreply.github.com" + + - name: "Stage files for commit" + ansible.builtin.command: + cmd: "git add ." + chdir: "{{ rust_project_dir }}" + + - name: "Check staged files" + ansible.builtin.command: + cmd: "git diff --staged" + chdir: "{{ rust_project_dir }}" + register: "staged_changes" + + - name: "Commit changes" + ansible.builtin.command: + cmd: "git commit -m 'feat: New generated content' -m '{{ zuul.change_message }}' -m 'Changes are triggered by {{ zuul.change_url }}'" + chdir: "{{ rust_project_dir }}" + register: "commit" + when: + # Only commit when there is anything to commit + - "staged_changes.stdout | length > 0" + + - name: "Format patch" + ansible.builtin.command: + cmd: "git format-patch -1 --output {{ patch_path }}" + chdir: "{{ rust_project_dir }}" + when: + # Only prepare when there is anything to commit + - "commit is defined" + - "commit.changed" diff --git a/playbooks/rust/post.yaml b/playbooks/rust/post.yaml new file mode 100644 index 0000000..10a2ef4 --- /dev/null +++ b/playbooks/rust/post.yaml @@ -0,0 +1,72 @@ +--- +- hosts: all + vars: + archive_path: "{{ ansible_user_dir }}/archive.tar.gz" + tasks: + - name: "Ensure local output dirs" + delegate_to: localhost + ansible.builtin.file: + path: "{{ zj_output_dir }}" + state: directory + mode: 0755 + with_items: + - "{{ zuul.executor.work_root }}/artifacts" + loop_control: + loop_var: zj_output_dir + + - name: "Archive generated code" + community.general.archive: + path: "{{ codegenerator_base_dir }}/wrk/rust" + dest: "{{ archive_path }}" + + - name: "Collect archived generated code" + ansible.posix.synchronize: + dest: "{{ zj_output.dest }}" + mode: pull + src: "{{ zj_output.src }}" + verify_host: true + owner: false + group: false + loop: + - dest: "{{ zuul.executor.work_root }}/artifacts/" + src: "{{ archive_path }}" + loop_control: + loop_var: zj_output + + - name: "Return source code artifact to Zuul" + zuul_return: + data: + zuul: + artifacts: + - name: "Code changes" + url: "artifacts/archive.tar.gz" + metadata: + type: "content" + + - name: Find generated patch file + ansible.builtin.stat: + path: "{{ patch_path }}" + register: patch_stat + + - name: "Collect patch file" + ansible.posix.synchronize: + dest: "{{ zuul.executor.work_root }}/artifacts/" + mode: pull + src: "{{ patch_path }}" + verify_host: true + owner: false + group: false + when: + - "patch_stat.stat.exists" + + - name: "Return patch artifact to Zuul" + zuul_return: + data: + zuul: + artifacts: + - name: "Git patch" + url: "artifacts/{{ zuul.change }}.patch" + metadata: + type: "rust_patch" + when: + - "patch_stat.stat.exists" diff --git a/playbooks/rust/propose-github.yaml b/playbooks/rust/propose-github.yaml new file mode 100644 index 0000000..0a4bf9c --- /dev/null +++ b/playbooks/rust/propose-github.yaml @@ -0,0 +1,61 @@ +--- +- name: "Propose generated code changes upstream" + hosts: localhost + vars: + patch_file: "{{ zuul.executor.work_root }}/{{ zuul.change }}.patch" + branch_name: "codegenerator_{{ zuul.change }}" + tasks: + + - name: "Check execution context" + when: "zuul.branch is not defined" + fail: + msg: "This playbook must be run in a branch-based pipeline (e.g., 'promote')." + + - name: "Download docs archive" + ansible.builtin.include_role: + name: download-artifact + vars: + # download_artifact_job: provided by zuul job + download_artifact_api: "https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}" + download_artifact_type: + - rust_patch + download_artifact_pipeline: gate + + - name: "Check git patch presense" + ansible.builtin.stat: + path: "{{ patch_file }}" + register: "git_patch_stat" + + - name: "Process patch" + when: + - "git_patch_stat.stat.exists" + block: + + - name: "Checkout target repository" + ansible.builtin.git: + repo: "{{ rust_sdk_git_repo }}" + dest: "{{ rust_project_dir }}" + + - name: "Checkout new branch" + ansible.builtin.command: "git checkout -b {{ branch_name }}" + args: + chdir: "{{ ansible_user_dir }}/openstack" + + - name: "Try to apply git patch" + ansible.builtin.command: "git apply --reject --ignore-space-change {{ git_patch_stat.stat.path }}" + args: + chdir: "{{ ansible_user_dir }}/openstack" + register: "patch_applied" + failed_when: false + + - name: "Push changes" + ansible.builtin.command: "git push" + args: + chdir: "{{ ansible_user_dir }}/openstack" + when: "patch_applied.changed" + register: "change_pushed" + + - name: "Inform how to open PR" + ansible.builtin.debug: + msg: "Please follow the link https://github.com/gtema/openstack/pull/new/{{ branch_name }} to create new pull request" + when: "change_pushed.changed" diff --git a/roles/codegenerator/tasks/main.yaml b/roles/codegenerator/tasks/main.yaml index 0541b57..b851bf0 100644 --- a/roles/codegenerator/tasks/main.yaml +++ b/roles/codegenerator/tasks/main.yaml @@ -1,5 +1,18 @@ --- -- name: Invoke openstack-codegenerator +- name: "Read service metadata" + ansible.builtin.slurp: + src: "{{ codegenerator_base_dir }}/{{ codegenerator_metadata }}" + register: "metadata_content" + +- ansible.builtin.set_fact: + metadata: "{{ metadata_content.content | b64decode | from_yaml }}" + +- name: "Check presense of openapi file (first one)" + ansible.builtin.stat: + path: "{{ codegenerator_base_dir }}/{{ metadata.resources | dict2items | map(attribute='value') | map(attribute='spec_file') | first }}" + register: "openapi_stat" + +- name: "Invoke openstack-codegenerator" args: chdir: "{{ codegenerator_base_dir }}" executable: "/bin/bash" @@ -14,3 +27,5 @@ {% if codegenerator_service_type is defined %} --service {{ codegenerator_service_type }} {% endif %} + when: + - "openapi_stat.stat.exists" diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 26ff430..849dc9d 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -32,3 +32,6 @@ - codegenerator-openapi-shared-file-system-tips-with-api-ref - codegenerator-tox-publish-openapi-specs - codegenerator-rust-all + post: + jobs: + - codegenerator-propose-rust-openstack-change diff --git a/zuul.d/rust.yaml b/zuul.d/rust.yaml index 1a79b66..114da44 100644 --- a/zuul.d/rust.yaml +++ b/zuul.d/rust.yaml @@ -8,6 +8,8 @@ timeout: 1800 pre-run: - playbooks/codegenerator/pre.yaml + post-run: + - playbooks/rust/post.yaml vars: codegenerator_service_metadata_target_map: - service: "block-storage" @@ -19,9 +21,11 @@ - service: "identity" metadata: "metadata/identity_metadata.yaml" targets: ["rust-sdk", "rust-cli"] - - service: "image" - metadata: "metadata/image_metadata.yaml" - targets: ["rust-sdk", "rust-cli"] + # https://review.opendev.org/c/openstack/glance/+/882498 screwed us. + # Disable build for glance until we find way to deal with that + # - service: "image" + # metadata: "metadata/image_metadata.yaml" + # targets: ["rust-sdk", "rust-cli"] - service: "load-balancer" metadata: "metadata/load-balancer_metadata.yaml" targets: ["rust-sdk", "rust-cli"] @@ -38,6 +42,8 @@ - job: name: codegenerator-rust-all parent: codegenerator-rust-base + # It takes a while to compile the project + timeout: 3600 description: | Generate Rust SDK/CLI dependencies: @@ -62,5 +68,30 @@ pre-run: playbooks/openapi/fetch.yaml run: playbooks/rust/all.yaml vars: - codegenerator_base_dir: "{{ zuul.project.src_dir }}" + codegenerator_base_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" openapi_dest: "{{ codegenerator_base_dir }}/wrk/openapi_specs" + patch_path: "{{ ansible_user_dir }}/{{ zuul.change }}.patch" + rust_sdk_git_repo: "https://github.com/gtema/openstack" + rust_project_dir: "{{ ansible_user_dir }}/openstack" + +- job: + name: codegenerator-propose-rust-openstack-change + description: | + Propose changes to the Rust OpenStack project with the code newly + generated in this change. + + This job is applying the git patch created by the + `codegenerator-rust-all` job. It accesses target repository using SSH + and a public key of the project (`curl + https://zuul.opendev.org/api/tenant/openstack/project-ssh-key/openstack/codegenerator.pub`) + being added as a deploy key in GitHub repository with read/write access. + Pull Request itself is not opened, since it requires additionally API + token what would require hardcoding secret (and GitHub does everything + possible not to have long lasting tokens). + post-review: true + run: playbooks/rust/propose-github.yaml + nodeset: + nodes: [] + vars: + rust_sdk_git_repo: "git@github.com:gtema/openstack.git" + rust_project_dir: "{{ ansible_user_dir }}/openstack"