From 17e0a7703f78a308dd68939bb1048f8ed41df760 Mon Sep 17 00:00:00 2001 From: Ramy Asselin Date: Thu, 24 Sep 2015 16:13:25 -0700 Subject: [PATCH] Add documentation for third-party-ci setup Add the initial documentation for setting common-ci using masterless puppet. Change-Id: I6a3b7493c808d66141a215016944d314cec1f0b0 --- contrib/README.md | 366 ++++++++++++++++++ contrib/hiera.yaml | 9 + contrib/log_server_data.yaml | 11 + contrib/log_server_site.pp | 33 ++ ...ci_hiera.yaml => single_node_ci_data.yaml} | 8 +- 5 files changed, 423 insertions(+), 4 deletions(-) create mode 100644 contrib/README.md create mode 100644 contrib/hiera.yaml create mode 100644 contrib/log_server_data.yaml create mode 100644 contrib/log_server_site.pp rename contrib/{single_node_ci_hiera.yaml => single_node_ci_data.yaml} (98%) diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..bcba9f2 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,366 @@ +# OpenStack Third-Party CI + +These instructions provide a **Third Party Testing** solution using the same +tools and scripts used by the OpenStack Infrastructure 'Jenkins' CI +system. + +If you are setting up a similar system for use outside of OpenStack, many of these +steps are still valid, while others can be skipped. These will be mentioned within +each step. + +If you are creating a third-party CI system for use within OpenStack, +you'll need to familiarize yourself with the contents of the +[third party manual](http://docs.openstack.org/infra/system-config/third_party.html), +and in particular you'll need to [create a service account] +(http://docs.openstack.org/infra/system-config/third_party.html#creating-a-service-account). + +## Overview + +This CI solution uses a few open-source tools: + +* [Jenkins](http://docs.openstack.org/infra/system-config/jenkins.html) - an open-source continuous integration server. + +* [Zuul](http://docs.openstack.org/infra/system-config/zuul.html) - a project gating system + +* [Nodepool](http://docs.openstack.org/infra/system-config/nodepool.html)- a node management system for testing + +* [Jenkins Job Builder](http://docs.openstack.org/infra/system-config/jjb.html) - a tool to manage jenkins job definitions + +* [os-loganalyze](http://git.openstack.org/cgit/openstack-infra/os-loganalyze/) - a tool to faciltate + browsing, sharing, and filtering log files by log level. + + +The follow steps will help you integrate and deploy the first 4 tools on a single +host. An initial system with 8GB RAM, 4CPUs, 80GB HD should be sufficient, +running Ubuntu 14.04. + +A second host will be used to store the log files and create a public log server to +host the static log files generated by jenkins jobs. This log server is an Apache +server serving log files stored on disk or on a Swift service. +It is hosted on a seperate server because it usually +needs to be publicly accessible to share job results whereas the rest of the CI +system can be located behind a firewall or within a VPN. At the end of a Jenkins Job, +`publishers` will scp log files from the jenkins slave to the log server or upload +to the Swift Service. + +The system requires two external resources: + +* A source for Nodepool nodes. This is a service that provides virtual machines + (e.g. OpenStack - Nova) or bare metal (e.g. OpenStack - Ironic) that nodepool + will manage as a pool of Jenkins slaves that will run the actual CI jobs. + You can use a public or private OpenStack cloud, or even run your own + [devstack](https://git.openstack.org/cgit/openstack-dev/devstack/) to get started. + + +* A Gerrit server (for OpenStack users, this is provided to you at review.openstack.org) + Zuul will listen to the Gerrit even stream to decide which jobs to run when it receives + a desired event. Zuul will also post a comment with results to this Gerrit with the + job results along with a link to the related log files. + +These instructions are for a 'masterless' puppet setup, which is the simplest +version to set up for those not familiar with puppet. + + +## Install and Configure Puppet + +On each host, you will need to install and configure puppet. + +### Install Puppet + +Puppet is a tool to automate the installation of servers by defining the desired +end state. You can quickly install puppet along with basic tools (such as pip and git) +using this script: + + wget https://git.openstack.org/cgit/openstack-infra/system-config/plain/install_puppet.sh + sudo bash install_puppet.sh + +### Install Puppet Modules + +You can get the latest version of the puppet modules needed using this script. + + git clone https://git.openstack.org/openstack-infra/system-config + cd system-config + sudo ./install_modules.sh + cd .. + +This script will install all the puppet modules used by upstream to +`/etc/puppet/modules`. In many cases, these are git cloned, and running +the `install_modules.sh` script again will update them to the latest version. +This script uses `modules.env` as its configuration input. + +### Configure Masterless Puppet + +First, it is useful to save the history, so set up a git repo as root user: + + cd /etc/puppet + git init + +You will be configuring 3 puppet files. The first is `site.pp` which is the top +level entry point for puppet to start managing the node. The second is a `hiera.yaml` +which configures Puppet Hiera to store local configurations and secrets +such as passwords and private keys, and finaly some `yaml` files which store the +actual configurations and secrets. + +First, set up these 3 files by starting with the samples provided. For each host, +select the corresponding `single_node_ci*` or `log_server*` files. + + cp /etc/puppet/modules/openstackci/contrib/hiera.yaml /etc/puppet + + cp /etc/puppet/modules/openstackci/contrib/single_node_ci_site.pp /etc/puppet/manifests/site.pp + cp /etc/puppet/modules/openstackci/contrib/single_node_ci_data.yaml /etc/puppet/environments/common.yaml + + OR + + cp /etc/puppet/modules/openstackci/contrib/log_server_site.pp /etc/puppet/manifests/site.pp + cp /etc/puppet/modules/openstackci/contrib/log_server_data.yaml /etc/puppet/environments/common.yaml + +At this point, you should not need to modify either of the first two files. +Modify `/etc/puppet/environments/common.yaml` as you need using the parameter +documentation described in `../manifests/single_node_ci.pp` or +`../manifests/logserver.pp` (which is +the top level puppet class that is used in `site.pp`). + +Once completed, you should commit these 3 files to the `/etc/puppet` git repo. + +# Setup the log server + +Setup the log server first, because it is simpler, and its FQDN (or IP address) +will be needed when setting up the CI server. + +While setting up jenkins_ssh_public_key in `common.yaml` it is important that +the same ssh key pair is used when setting up the CI server in the next step. +This is the ssh key that Jenkins will use to scp files. + +At this point you are ready to invoke Puppet for the first time. Puppet needs to +be run as root. + + sudo puppet apply --verbose /etc/puppet/manifests/site.pp + +You can simulate a jenkins file upload using: + + scp -i $JENKINS_SSH_PUBLIC_KEY -o StrictHostKeyChecking=no $your-log-file jenkins@:/srv/static/logs/ + +You should now be able to see the file you uploaded at `http:///$your-log-file` + +# Setup the CI server + +## Create an Initial 'project-config' Repository + +Setting up a CI system consists of two major operational aspects. The first is +system configuration, which focuses on the installation and deployment of the +services, including any ssh keys, credentials, databases, etc., and ensure all +system components are able to interact together. This portion is +performed by a System Administrator. + +The second is project configuration, which includes the configuration files +that the services use to perform the desired project-specific operations. + +The instructions provided here are mainly focused on the system configuration aspect. +However, they need an initial set of project configurations in order to work. These +project configurations are provided via a git URL to a `project-config` repository. +Before moving on, create an initial `project-config` repository. You can start with this +[project-config-example](https://git.openstack.org/cgit/openstack-infra/project-config-example/) +following the instructions provided in its README.md. While tailored for OpenStack users, +the instructions provided will help non-OpenStack users also start with this repository. +After your system is deployed, you can make further changes to the `project-config` +repository to continuously tailor it to your needs. + +## Add 'jenkins' to your host name + +Add 'jenkins' to your /etc/hosts file so that Apache (which will be installed by +the puppet scripts) is happy. This is needed because the scripts will install +multiple services on a single node. For example: + + head -n 1 /etc/hosts + 127.0.0.1 localhost jenkins + +## Run masterless Puppet + +At this point you are ready to invoke Puppet for the first time. Puppet needs to +be run as root. + + sudo puppet apply --verbose /etc/puppet/manifests/site.pp + +Puppet will install nodepool, jenkins, zuul, jenkins jobs builder, etc. + +Your `project-config` repository will be cloned to /etc/project-config, and the puppet scripts +will use these configuration files located in this folder. Do not update these files directly. +Instead, you should update them from a clone on a dev host, merge the changes to master, and push +them to the same git remote location. Puppet will always pull down the latest version of master +from the git remote and use that to update services. + +If you get the following error, manually run the failed `jenkins-jobs update` command with +the arguments specified in the error message as root. This is caused by a bug in the puppet +scripts where Jenkins is not yet running when Jenkins Job Builder attempts to load the +Jenkins jobs. + + Notice: /Stage[main]/Jenkins::Job_builder/Exec[jenkins_jobs_update]/returns: jenkins.JenkinsException: Error in request: [Errno 111] Connection refused + Notice: /Stage[main]/Jenkins::Job_builder/Exec[jenkins_jobs_update]/returns: INFO:jenkins_jobs.builder:Cache saved + Error: /Stage[main]/Jenkins::Job_builder/Exec[jenkins_jobs_update]: Failed to call refresh: jenkins-jobs update --delete-old /etc/jenkins_jobs/config returned 1 instead of one of [0] + Error: /Stage[main]/Jenkins::Job_builder/Exec[jenkins_jobs_update]: jenkins-jobs update --delete-old /etc/jenkins_jobs/config returned 1 instead of one of [0] + + +## Restart apache if necessary + +There are some known issues with Puppet automation. If you get the following error: + + AH00526: Syntax error on line 21 of /etc/apache2/sites-enabled/50-.conf: + Invalid command 'RewriteEngine', perhaps misspelled or defined by a module not included in the server configuration + +A simple restart works around the issue: + + sudo service apache2 restart + +## Start zuul + +We'll start zuul first: + + sudo service zuul start + sudo service zuul-merger start + +You should see 2 zuul-server processes and 1 zuul-merger process + + ps -ef | grep zuul + zuul 5722 1 2 18:13 ? 00:00:00 /usr/bin/python /usr/local/bin/zuul-server + zuul 5725 5722 0 18:13 ? 00:00:00 /usr/bin/python /usr/local/bin/zuul-server + zuul 5741 1 2 18:13 ? 00:00:00 /usr/bin/python /usr/local/bin/zuul-merger + +You can view the log files for any errors: + + view /var/log/zuul/zuul.log + +Most zuul files are located in either. They should not need to be modified directly, but +are useful to help identify root causes: + + /var/lib/zuul + /etc/zuul + +## Start nodepool + +The first time starting nodepool, it's recommended to manually build the image +to aid in debugging any issues: + + + sudo su - nodepool + + # Ensure the NODEPOOL_SSH_KEY variable is in the environment + # Otherwise nodepool won't be able to ssh into nodes based + # on the image built manually using these instructions + source /etc/default/nodepool + + # In the command below references one of the + # images defined in your project-config/nodepool/nodepool.yaml + # file as the 'name' field in the section 'diskimages'. + nodepool image-build + +If you run into issues building the image, the [documentation provided +here can help you debug](https://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/elements/README.rst) + + +After you have sucessfully built an image, manually upload it to the provider to ensure +provider authentication and image uploading work: + + nodepool image-upload all + +Once successful, you can start nodepool. +(Note that if you don't yet have an image, this is one of the first actions nodepool +will do when it starts, before creating any nodes): + + sudo service nodepool start + +You should see at least one process running. In particular: + + ps -ef | grep nodepool + nodepool 5786 1 28 18:14 ? 00:00:01 /usr/bin/python /usr/local/bin/nodepoold -c /etc/nodepool/nodepool.yaml -l /etc/nodepool/logging.conf + +## Setup Jenkins + +First Restart Jenkins so that plugins will be fully installed: + + sudo service jenkins restart + +Then open the Jenkins UI to finish manual configuration steps. + +Enable Gearman, which is the Jenkins plugin zuul uses to queue jobs: + + http://:8080/ + Manage Jenkins --> Configure System + Under "Gearman Plugin Config" Check the box "Enable Gearman" + Click "Test Connection" It should return success if zuul is running. + +Enable ZMQ Event Publisher, which is how nodepool is notified of Jenkin +slaves status events: + + http://:8080/ + Manage Jenkins --> Configure System + Under "ZMQ Event Publisher" + Check the box "Enable on all Jobs" + +## Securing Jenkins (optional) + +By default, Jenkins is installed with security disabled. While this is fine +for development environments where external access to Jenkins UI is restricted, +you are strongly encouraged to enable it. You can skip this step and do it +at a later time if you wish: + +Create a jenkins 'credentials': + + http://:8080/ + Manage Jenkins --> Add Credentials --> SSH Username with private key + Username 'jenkins' + Private key --> From a file on Jenkins master + "/var/lib/jenkins/.ssh/id_rsa" + --> Save + +Save the credential uuid in your hiera data: + + sudo su jenkins + cat /var/lib/jenkins/credentials.xml | grep "" + Copy the id to the 'jenkins_credentials_id' value in /etc/puppet/environments/common.yaml + +Enable basic Jenkins security: + + http://:8080/ + Manage Jenkins --> Configure Global Security + Check "Enable Security" + Under "Security Realm" + Select Jenkin's own user database + Uncheck allow users to sign up + Under "Authorization" select "logged-in users can do anything" + + Create a user 'jenkins' + Choose a password. + check 'Sign up' + Save the password to the 'jenkins_password' value in /etc/puppet/environments/common.yaml + +Get the new 'jenkins' user API token: + + http://:8080/ + Manage Jenkins --> People --> Select user 'jenkins' --> configure --> Show API Token + Save this token to the 'jenkins_api_key' value in /etc/puppet/environments/common.yaml + +Reconfigure your system to use Jenkins security settings stored in +`/etc/puppet/environments/common.yaml` + + sudo puppet apply --verbose /etc/puppet/manifests/site.pp + +# Updating your masterless puppet hosts + +Any time you check-in changes to your `project-config` repo, make changes to the +hiera data (`/etc/puppet/environments/common.yaml`), or update +the puppet files (in /etc/puppet/modules, either manually or via the `install_modules.sh` script), +run the same puppet command to update the host. + + sudo puppet apply --verbose /etc/puppet/manifests/site.pp + +If you need to change the git url in your `project-config` or any other git urls +in your `common.yaml`, delete the respective repository, e.g. +`/etc/project-config`, and puppet will reclone it from the new location when the above `puppet apply` +command is reinvoked. + +Note that it is safe, and expected, to rerun the above `puppet apply` command. Puppet will +update the configuration of the host as described in the puppet classes. This means that if +you delete or modify any files managed by puppet, rerunning the `puppet apply` command will +restore those settings back to the specified state (and remove your local changes for better or worse). +You could even run the `puppet apply` command as a cron job to enable continuous deployment +in your CI system. diff --git a/contrib/hiera.yaml b/contrib/hiera.yaml new file mode 100644 index 0000000..49f8b3a --- /dev/null +++ b/contrib/hiera.yaml @@ -0,0 +1,9 @@ +--- +:backends: + - yaml +:logger: console +:hierarchy: + - common + +:yaml: + :datadir: /etc/puppet/environments diff --git a/contrib/log_server_data.yaml b/contrib/log_server_data.yaml new file mode 100644 index 0000000..27fed0c --- /dev/null +++ b/contrib/log_server_data.yaml @@ -0,0 +1,11 @@ +# See parameter documetation inside ../manifests/single_node_ci.pp +# Fields commented out have reasonable default values +domain: your-domain.example.com +jenkins_ssh_public_key: your-jenkins-public-key-no-whitespace +#swift_authurl: +#swift_user: +#swift_key: +#swift_tenant_name: +#swift_region_name: +#swift_default_container: + diff --git a/contrib/log_server_site.pp b/contrib/log_server_site.pp new file mode 100644 index 0000000..df7ccf3 --- /dev/null +++ b/contrib/log_server_site.pp @@ -0,0 +1,33 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# +# A sample puppet node configuration that installs and configures a server +# that hosts log files that are viewable in a browser. +# Note that using swift is optional and the defaults provided disable its +# usage. + +node default { + class { '::openstackci::logserver': + domain => hiera('domain'), + jenkins_ssh_key => hiera('jenkins_ssh_public_key'), + swift_authurl => hiera('swift_authurl', ''), + swift_user => hiera('swift_user', ''), + swift_key => hiera('swift_key', ''), + swift_tenant_name => hiera('swift_tenant_name', ''), + swift_region_name => hiera('swift_region_name', ''), + swift_default_container => hiera('swift_default_container', ''), + } +} diff --git a/contrib/single_node_ci_hiera.yaml b/contrib/single_node_ci_data.yaml similarity index 98% rename from contrib/single_node_ci_hiera.yaml rename to contrib/single_node_ci_data.yaml index a11656a..317ce3c 100644 --- a/contrib/single_node_ci_hiera.yaml +++ b/contrib/single_node_ci_data.yaml @@ -9,16 +9,16 @@ project_config_repo: http://your-project-config-repo.example.com/project-config- jenkins_ssh_private_key: | -----BEGIN RSA PRIVATE KEY----- jenkins private key -jenkins_ssh_public_key: your-jenkins-public-key-no-whitespace -----END RSA PRIVATE KEY----- +jenkins_ssh_public_key: your-jenkins-public-key-no-whitespace #gerrit_server: review.openstack.org -gerrit_user: gerrit-user -gerrit_user_ssh_public_key: gerrit-public-key-no-whitespace +#gerrit_ssh_host_key: +gerrit_user: your-gerrit-user gerrit_user_ssh_private_key: | -----BEGIN RSA PRIVATE KEY----- gerrit private key -----END RSA PRIVATE KEY----- -#gerrit_ssh_host_key: +gerrit_user_ssh_public_key: gerrit-public-key-no-whitespace git_email: your-email@example.com git_name: Your Name log_server: logs.example.com