Add Gitlab CI jobs

Without this patch the control repository does not contain any scripts
to execute CI jobs for Puppet code deployment.  This patch addresses the
problem by adding a Gitlab CI job configuration with three jobs defined
across two stages.  The test stage executes first with a Syntax and a
Lint job executing in parallel.  If an environment branch has been
pushed, one matching `developemnt`, `testing`, `production` or starting
with the prefix `playground`, the deploy stage is executed.

The deploy stage requires a Gitlab secret environment variable,
PUPPET_TOKEN.  This environment variable is the puppet access token
configured with Code Manager.

The test stage runs the following syntax checks:

 * yaml files (*.yml, *.yaml)
 * json files (*.json)
 * bash scripts (*.sh)
 * ruby scripts (*.rb)
 * puppet manifests (*.pp)
 * erb files (*.erb)

The behavior of the checks are to check only files modified relative to
a base branch, defaulting to `production`.  This avoids running syntax
checks against files which have not been modified by the merge request.
The check uses `git diff --name-status` to identify changed files across
multiple commits in a topic branch.

The lint checks rely on bundler and the Gemfile to install numerous lint
checks.

Gem libraries are installed in a per-job location in $HOME.  This is an
intentional compromise between installing into a system GEM_PATH
element, which would create library conflicts with other jobs, and the
job workspace, which would cause gem libraries to be installed from the
network on each job invocation.
This commit is contained in:
Jeff McCune 2017-06-14 16:37:20 -04:00
parent ac9785273a
commit de08460ef6
5 changed files with 266 additions and 0 deletions

39
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,39 @@
---
stages:
- test
- deploy
# Jobs are defined below
Syntax Check:
tags:
- puppet agent
stage: test
script:
- scripts/syntax_check.sh
Lint Check:
tags:
- puppet agent
stage: test
script:
- scripts/lint_check.sh
# Unless TCP port 8170 is open to the PE master, this job must execute on the
# PE Monolithic Master itself.
Deploy to Puppet:
tags:
- puppet monolithic master
stage: deploy
variables:
URL: https://puppet:8170/code-manager
only:
- development
- testing
- production
- /^playground/
script:
- scripts/puppet_deploy.sh
# vim:tabstop=2
# vim:shiftwidth=2
# vim:expandtab

19
Gemfile Normal file
View File

@ -0,0 +1,19 @@
source ENV['GEM_SOURCE'] || 'https://rubygems.org'
if puppetversion = ENV['PUPPET_GEM_VERSION']
gem 'puppet', puppetversion, require: false
else
gem 'puppet', require: false
end
gem 'puppet-lint', '~> 2.0'
gem 'puppet-lint-absolute_classname-check'
gem 'puppet-lint-alias-check'
gem 'puppet-lint-empty_string-check'
gem 'puppet-lint-file_ensure-check'
gem 'puppet-lint-file_source_rights-check'
gem 'puppet-lint-leading_zero-check'
gem 'puppet-lint-trailing_comma-check'
gem 'puppet-lint-undef_in_function-check'
gem 'puppet-lint-unquoted_string-check'
gem 'puppet-lint-variable_contains_upcase'

72
scripts/lint_check.sh Executable file
View File

@ -0,0 +1,72 @@
#!/bin/bash
# Error out if there are any failures
set -e
set -o pipefail
set -u
# Notes
# xargs -P2 is used to run 2 parallel processes at once. This speeds up
# performance on multi-core systems.
if [ -e /proc/cpuinfo ]; then
cores=$(awk 'BEGIN { c = 0 }; $1 == "processor" { c++ }; END { print c }' /proc/cpuinfo)
else
cores=2
fi
# Use Puppet Enterprise Ruby to check ruby and yaml files
export PATH="/opt/puppetlabs/puppet/bin:/opt/puppetlabs/bin:$PATH"
# Localize Gems on a per-job basis to prevent conflicts
gem_home="$(gem env gempath | cut -d: -f1)"
# Trim off the leading part of $HOME
gem_suffix=${gem_home##*/.gem/}
# Set GEM_HOME to a job specific location
export GEM_HOME="${HOME}/jobs/${CI_JOB_NAME:-lint}/gem/${gem_suffix}"
# If we need to install a gem, do so into HOME
# e.g. /home/gitlab-runner/.gem/ruby/2.1.0
export PATH="${GEM_HOME}/bin:$PATH"
echo '######## BEGIN DEPENDENCY SETUP #########'
# Display the gem environment
gem env
if ! (which bundle 2>&1 >/dev/null); then
gem install bundler --no-ri --no-rdoc
fi
# List the files changes from $BASEBRANCH on stdout
files_changed() {
# File status flags:
# M modified - File has been modified
# C copy-edit - File has been copied and modified
# R rename-edit - File has been renamed and modified
# A added - File has been added
# D deleted - File has been deleted
# U unmerged - File has conflicts after a merge
git diff --name-status "${BASEBRANCH:=production}" \
| awk '$1 ~ /^[MCRA]$/' \
| cut -f2-
}
# Install dependencies
bundle install
echo '######## END DEPENDENCY SETUP #########'
echo
echo
echo '######## BEGIN LINT CHECKS #########'
# Lint only the manifest files changed
files_changed \
| awk '/manifests\/.*\.(pp)$/' \
| xargs --no-run-if-empty -t -P$cores -n1 \
bundle exec puppet-lint
echo '######## END LINT CHECKS #########'
# vim:tabstop=2
# vim:shiftwidth=2
# vim:expandtab

60
scripts/puppet_deploy.sh Executable file
View File

@ -0,0 +1,60 @@
#!/bin/bash
set -u
export PATH="/opt/puppetlabs/bin:$PATH"
# Allow these environment variable to be overriden
: ${URL:='https://puppet:8170/code-manager'}
# CI_BUILD_REF_NAME is a variable set by gitlab
: ${ENVIRONMENT:="$CI_BUILD_REF_NAME"}
err() {
echo "$1" >&2
}
if [ -z "${PUPPET_TOKEN:-}" ]; then
err "ERROR: PUPPET_TOKEN environment variable must be set!"
err "SUGGESTION: Did you push to origin instead of upstream?"
err "PUPPET_TOKEN must be set as an environment variable in CI"
exit 1
fi
if ! [ -x /opt/puppetlabs/bin/puppet-code ]; then
err "ERROR: /opt/puppetlabs/bin/puppet-code does not exist"
err "SUGGESTION: Install the puppet client tools"
err "https://docs.puppet.com/pe/2016.4/install_pe_client_tools.html#install-on-a-linux-workstation"
exit 2
fi
# Save the token to a temporary file so we can use it with puppet code deploy
scratch="$(mktemp -d)"
remove_scratch() {
[ -e "${scratch:-}" ] && rm -rf "$scratch"
}
trap remove_scratch EXIT
# Subsequent calls to mktemp should be inside our scratch dir
export TMPDIR="$scratch"
tokenfile="$(mktemp)"
echo -n "$PUPPET_TOKEN" > "$tokenfile"
# Turn on debug logging after the token has been written to the file system
set -x
# Deploy the code
puppet-code deploy \
--service-url "$URL" \
--token-file "$tokenfile" \
--wait "${ENVIRONMENT}"
rval=$?
set +x
if [ $rval -ne 0 ]; then
echo "ERROR: puppet-code deploy failed with exit code $rval" >&2
exit $rval
fi
echo "Exiting with exit value $rval"
exit $rval
# vim:tabstop=2
# vim:shiftwidth=2
# vim:expandtab

76
scripts/syntax_check.sh Executable file
View File

@ -0,0 +1,76 @@
#!/bin/bash
# Error out if there are any failures
set -e
set -o pipefail
set -u
# Notes
# xargs -P2 is used to run 2 parallel processes at once. This speeds up
# performance on multi-core systems.
if [ -e /proc/cpuinfo ]; then
cores=$(awk 'BEGIN { c = 0 }; $1 == "processor" { c++ }; END { print c }' /proc/cpuinfo)
else
cores=2
fi
# Use Puppet Enterprise Ruby to check ruby and yaml files
export PATH="/opt/puppetlabs/puppet/bin:$PATH"
# List the files changes from $BASEBRANCH on stdout
files_changed() {
# File status flags:
# M modified - File has been modified
# C copy-edit - File has been copied and modified
# R rename-edit - File has been renamed and modified
# A added - File has been added
# D deleted - File has been deleted
# U unmerged - File has conflicts after a merge
git diff --name-status "${BASEBRANCH:=production}" \
| awk '$1 ~ /^[MCRA]$/' \
| cut -f2-
}
# Check the Puppetfile
echo -n "Checking Puppetfile ... "
ruby -c Puppetfile
files_changed \
| awk '/\.(sh)$/' \
| xargs --no-run-if-empty -t -P$cores -n1 \
bash -n
# Check all YAML files
# See: http://stackoverflow.com/questions/3971822/yaml-syntax-validator
files_changed \
| awk '/\.(yml|yaml)$/' \
| xargs --no-run-if-empty -t -P$cores -n1 \
ruby -r yaml -e 'YAML.load_file(ARGV[0])'
# Check all JSON files
files_changed \
| awk '/\.(json)$/' \
| xargs --no-run-if-empty -t -P$cores -n1 \
ruby -r json -e 'JSON.load(File.read(ARGV[0]))'
files_changed \
| awk '/\.(rb)$/' \
| xargs --no-run-if-empty -t -P$cores -n1 \
ruby -c
# Check all erb files
files_changed \
| awk '/\.(erb)$/' \
| xargs -l --no-run-if-empty -t -P$cores -n1 \
bash -c 'erb -P -x -T- $0 | ruby -c'
# Check all Puppet manifest files
files_changed \
| awk '/manifests\/.*\.(pp)$/' \
| xargs --no-run-if-empty -t -P$cores -n1 \
puppet parser validate
# vim:tabstop=2
# vim:shiftwidth=2
# vim:expandtab