Merge branch 'drybjed-role-updates'

Author: drybjed

CHANGES.rst

@@ -14,7 +14,52 @@ The current role maintainer_ is drybjed.
 `debops.gitlab_runner master`_ - unreleased
 -------------------------------------------
 
-.. _debops.gitlab_runner master: https://github.com/debops/ansible-gitlab_runner/compare/v0.1.2...master
+.. _debops.gitlab_runner master: https://github.com/debops/ansible-gitlab_runner/compare/v0.2.0...master
+
+
+`debops.gitlab_runner v0.2.0`_ - 2017-08-16
+-------------------------------------------
+
+.. _debops.gitlab_runner v0.2.0: https://github.com/debops/ansible-gitlab_runner/compare/v0.1.2...v0.2.0
+
+Added
+~~~~~
+
+- Add improved autoscaling off-peak support. [azman0101]
+
+- Add a separate list of tags defined for shell executors that describe the
+  host in a greater detail. [drybjed_]
+
+Changed
+~~~~~~~
+
+- Fixed Ansible warnings during task execution. [drybjed_]
+
+- Update APT cache before first package installation. [azman0101, drybjed_]
+
+- Convert the Ansible local facts to a Python script. The GitLab Runner state
+  is stored in a separate, secure JSON file. [drybjed_]
+
+- Redesign the list of default GitLab Runner executors. The role will create
+  a 'shell' executor, unless Docker is detected in which case the 'shell'
+  executor will be disabled. Two Docker executors will be created, one
+  privileged and one unprivileged, with respective tags. [drybjed_]
+
+- If Docker is detected, the ``gitlab-runner`` user will be added to the
+  ``docker`` group to allow access to Docker containers. [drybjed_]
+
+- On hosts with Docker executors, number of concurrent jobs will depend on
+  number of vCPUs available. Hosts with 'shell' executor will be allowed to run
+  only 1 job at a time. [drybjed_]
+
+- The default 'shell' and privileged Docker executors will not be allowed to
+  run untagged jobs for security reasons. The default unprivileged Docker
+  executor can still run untagged jobs. [drybjed_]
+
+Fixed
+~~~~~
+
+- Fixed an error with missing Ansible facts breaking GitLab Runner registration. [drybjed_]
 
 
 `debops.gitlab_runner v0.1.2`_ - 2016-10-05

defaults/main.yml

@@ -107,7 +107,10 @@ gitlab_runner__shell: '/bin/bash'
 # .. envvar:: gitlab_runner__concurrent [[[
 #
 # Global number of jobs that can run concurrently on all configured runners.
-gitlab_runner__concurrent: '{{ ansible_processor_cores }}'
+gitlab_runner__concurrent: '{{ ansible_processor_vcpus
+                               if (ansible_local|d() and ansible_local.docker|d() and
+                                   (ansible_local.docker.installed|d()) | bool)
+                               else "1" }}'
 
                                                                    # ]]]
 # .. envvar:: gitlab_runner__domain [[[
@@ -159,6 +162,14 @@ gitlab_runner__environment: {}
 # GitLab Runner tags [[[
 # ----------------------
 
+# .. envvar:: gitlab_runner__shell_tags [[[
+#
+# List of tags automatically generated by the role, based on Ansible facts.
+# These tags are used with shell executors.
+gitlab_runner__shell_tags: '{{ lookup("template",
+                               "lookup/gitlab_runner__shell_tags.j2") }}'
+
+                                                                   # ]]]
 # .. envvar:: gitlab_runner__default_tags [[[
 #
 # List of default tags applied to the Runner at registration in GitLab CI.
@@ -182,6 +193,15 @@ gitlab_runner__group_tags: []
 #
 # List of per-host tags applied to the Runner at registration in GitLab CI.
 gitlab_runner__host_tags: []
+
+                                                                   # ]]]
+# .. envvar:: gitlab_runner__combined_tags [[[
+#
+# Combined list of system-wide tags for easier use in the role tasks.
+gitlab_runner__combined_tags: '{{ gitlab_runner__default_tags
+                                  + gitlab_runner__tags
+                                  + gitlab_runner__group_tags
+                                  + gitlab_runner__host_tags }}'
                                                                    # ]]]
                                                                    # ]]]
 # GitLab Runner instances [[[
@@ -200,7 +220,34 @@ gitlab_runner__host_tags: []
 # .. envvar:: gitlab_runner__default_instances [[[
 #
 # List of default GitLab Runner instances.
-gitlab_runner__default_instances: [ '{{ gitlab_runner__instance_shell }}' ]
+gitlab_runner__default_instances:
+
+  - name: '{{ ansible_hostname + "-shell" }}'
+    executor: 'shell'
+    run_untagged: False
+    state: '{{ "absent"
+               if (ansible_local|d() and ansible_local.docker|d() and
+                   (ansible_local.docker.installed|d()) | bool)
+               else "present" }}'
+
+  - name: '{{ ansible_hostname + "-docker" }}'
+    executor: 'docker'
+    tags: [ 'docker' ]
+    state: '{{ "present"
+               if (ansible_local|d() and ansible_local.docker|d() and
+                   (ansible_local.docker.installed|d()) | bool)
+               else "absent" }}'
+
+  - name: '{{ ansible_hostname + "-docker-root" }}'
+    executor: 'docker'
+    concurrent: '{{ ansible_processor_vcpus }}'
+    docker_privileged: True
+    run_untagged: False
+    tags: [ 'docker-privileged' ]
+    state: '{{ "present"
+               if (ansible_local|d() and ansible_local.docker|d() and
+                   (ansible_local.docker.installed|d()) | bool)
+               else "absent" }}'
 
                                                                    # ]]]
 # .. envvar:: gitlab_runner__instances [[[
@@ -219,14 +266,6 @@ gitlab_runner__group_instances: []
 #
 # List of GitLab Runner instances on a particular host.
 gitlab_runner__host_instances: []
-
-                                                                   # ]]]
-# .. envvar:: gitlab_runner__instance_shell [[[
-#
-# The default GitLab Runner instance.
-gitlab_runner__instance_shell:
-  name: '{{ gitlab_runner__fqdn }}'
-  executor: 'shell'
                                                                    # ]]]
                                                                    # ]]]
 # Custom file upload [[[

docs/getting-started.rst

@@ -39,9 +39,11 @@ Initial configuration
 ---------------------
 
 By default, ``debops.gitlab_runner`` will configure a single Runner instance
-which uses a shell executor. To use different executors like SSH or Docker, you
-need to provide additional configuration and ensure that required software
-(:command:`ssh`, Docker, Docker Machine, etc.) is installed if required.
+which uses a shell executor. If a Docker installation is detected via Ansible
+local facts, the role will disable the shell executor and configure two Docker
+executors - one unprivileged, and one privileged. The executors will have a set
+of tags that identify them, shell executors will have additional tags that
+describe the host's architecture, OS release, etc.
 
 The Runner instances can be configured with variables specified as the keys of
 the dictionary that holds the specific Runner configuration. If any required

tasks/main.yml

@@ -23,6 +23,14 @@
     generate_ssh_key: '{{ gitlab_runner__ssh_generate | bool }}'
     ssh_key_bits: '{{ gitlab_runner__ssh_generate_bits }}'
 
+- name: Allow Docker access for gitlab-runner user
+  user:
+    name: '{{ gitlab_runner__user }}'
+    groups: 'docker'
+    append: True
+  when: (ansible_local|d() and ansible_local.docker|d() and
+         (ansible_local.docker.installed|d()) | bool)
+
 - name: Copy custom files to GitLab Runner host
   copy:
     src: '{{ item.src | d(omit) }}'
@@ -45,6 +53,11 @@
     name: '{{ item }}'
     state: 'present'
     install_recommends: False
+    update_cache: True
+    cache_valid_time: '{{ ansible_local.core.cache_valid_time
+                          if (ansible_local|d() and ansible_local.core|d() and
+                              ansible_local.core.cache_valid_time|d())
+                           else "86400" }}'
   with_items: [ 'apt-transport-https', 'openssl', 'ca-certificates' ]
 
 - name: Add gitlab-runner APT key
@@ -79,12 +92,13 @@
   register: gitlab_runner__register_api
   when: gitlab_runner__api_url|d() and gitlab_runner__api_url
   failed_when: False
+  no_log: True
 
 - name: Register new GitLab Runners
   uri:
     url: '{{ (item.api_url | d(gitlab_runner__api_url)) + "/api/v1/runners/register.json" }}'
     method: 'POST'
-    body: 'token={{ item.token | d(gitlab_runner__token) }}&description={{ item.name | urlencode }}&tag_list={{ ((item.tags|d([]) + gitlab_runner__default_tags + gitlab_runner__tags + gitlab_runner__group_tags + gitlab_runner__host_tags) | unique | join(",")) | urlencode }}&run_untagged={{ gitlab_runner__run_untagged }}'
+    body: 'token={{ item.token | d(gitlab_runner__token) }}&description={{ item.name | urlencode }}&tag_list={{ ((item.tags|d([]) + (gitlab_runner__shell_tags if (item.executor == "shell") else []) + gitlab_runner__combined_tags) | unique | join(",")) | urlencode }}&run_untagged={{ item.run_untagged | d(gitlab_runner__run_untagged) }}'
     status_code: '200,201'
   register: gitlab_runner__register_new_instances
   with_flattened:
@@ -95,18 +109,19 @@
   when: (gitlab_runner__register_api.status|d() and gitlab_runner__register_api.status == 200 and
          (item.token | d() or gitlab_runner__token) and item.name and
          (item.state is undefined or item.state != 'absent') and
-         (ansible_local|d(True) and
-          (ansible_local.gitlab_runner is undefined or
-           item.name not in ansible_local.gitlab_runner.instances)) and
-        (gitlab_runner__run_untagged|d()))
+         (ansible_local is undefined or
+          (ansible_local|d() and (ansible_local.gitlab_runner is undefined or
+           (ansible_local.gitlab_runner|d() and ansible_local.gitlab_runner.instances is defined and
+            item.name not in ansible_local.gitlab_runner.instances)))))
 
-- name: Generate gitlab-runner configuration file
+- name: Generate GitLab Runner configuration files
   template:
-    src: 'etc/gitlab-runner/config.toml.j2'
-    dest: '/etc/gitlab-runner/config.toml'
+    src: 'etc/gitlab-runner/{{ item }}.j2'
+    dest: '/etc/gitlab-runner/{{ item }}'
     owner: 'root'
     group: 'root'
     mode: '0600'
+  with_items: [ 'config.toml', 'ansible.json' ]
 
 - name: Delete GitLab Runners if requested
   uri:
@@ -145,7 +160,7 @@
     owner: '{{ gitlab_runner__user }}'
     group: '{{ gitlab_runner__group }}'
     mode: '0700'
-  when: gitlab_runner__ssh_known_hosts
+  when: gitlab_runner__ssh_known_hosts|d()
 
 - name: Make sure the ~/.ssh/known_hosts file exists
   copy:
@@ -155,12 +170,12 @@
     group: '{{ gitlab_runner__group }}'
     mode: '0644'
     force: False
-  when: gitlab_runner__ssh_known_hosts
+  when: gitlab_runner__ssh_known_hosts|d()
 
 - name: Get list of already scanned host fingerprints
   shell: ssh-keygen -f {{ gitlab_runner__home }}/.ssh/known_hosts -F {{ item }} | grep -q '^# Host {{ item }} found'
   with_items: '{{ gitlab_runner__ssh_known_hosts }}'
-  when: gitlab_runner__ssh_known_hosts
+  when: gitlab_runner__ssh_known_hosts|d()
   register: gitlab_runner__register_known_hosts
   changed_when: False
   failed_when: False
@@ -185,4 +200,4 @@
     dest: '/etc/ansible/facts.d/gitlab_runner.fact'
     owner: 'root'
     group: 'root'
-    mode: '0600'
+    mode: '0755'

templates/etc/ansible/facts.d/gitlab_runner.fact.j2

@@ -1,26 +1,22 @@
-{% set gitlab_runner__tpl_instances = [] %}
-{% set gitlab_runner__tpl_instance_tokens = [] %}
-{% if gitlab_runner__register_new_instances|d() and gitlab_runner__register_new_instances.results %}
-{%   for item in gitlab_runner__register_new_instances.results %}
-{%     if item.item.name|d() and item.json|d() and item.json.token|d() %}
-{%       set _ = gitlab_runner__tpl_instances.append(item.item.name) %}
-{%       set _ = gitlab_runner__tpl_instance_tokens.append({ "name": item.item.name, "token": item.json.token }) %}
-{%     endif %}
-{%   endfor %}
-{% endif %}
-{% if ansible_local|d() and ansible_local.gitlab_runner|d() and ansible_local.gitlab_runner.instance_tokens|d() %}
-{%   for item in ansible_local.gitlab_runner.instance_tokens %}
-{%     if item.name not in gitlab_runner__tpl_instances %}
-{%       for element in (gitlab_runner__default_instances + gitlab_runner__instances + gitlab_runner__group_instances + gitlab_runner__host_instances) %}
-{%         if (item.name == element.name and (element.state is undefined or element.state != 'absent')) %}
-{%           set _ = gitlab_runner__tpl_instances.append(item.name) %}
-{%           set _ = gitlab_runner__tpl_instance_tokens.append({ "name": item.name, "token": item.token }) %}
-{%         endif %}
-{%       endfor %}
-{%     endif %}
-{%   endfor %}
-{% endif %}
-{
-"instance_tokens": {{ gitlab_runner__tpl_instance_tokens | to_nice_json }},
-"instances": {{ gitlab_runner__tpl_instances | to_nice_json }}
-}
+#!/usr/bin/env python
+
+# {{ ansible_managed }}
+
+from __future__ import print_function
+from json import load, loads, dumps
+from sys import exit
+import os
+
+output = {}
+state_file = '/etc/gitlab-runner/ansible.json'
+
+if os.path.isfile(state_file):
+    try:
+        fh = open(state_file)
+        output = load(fh)
+    except Exception:
+        pass
+
+output['installed'] = True
+
+print(dumps(output, sort_keys=True, indent=2))

templates/etc/gitlab-runner/ansible.json.j2

@@ -0,0 +1,27 @@
+{% set gitlab_runner__tpl_instances = [] %}
+{% set gitlab_runner__tpl_instance_tokens = [] %}
+{% if gitlab_runner__register_new_instances|d() and gitlab_runner__register_new_instances.results %}
+{%   for item in gitlab_runner__register_new_instances.results %}
+{%     if item.item.name|d() and item.json|d() and item.json.token|d() %}
+{%       set _ = gitlab_runner__tpl_instances.append(item.item.name) %}
+{%       set _ = gitlab_runner__tpl_instance_tokens.append({ "name": item.item.name, "token": item.json.token }) %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+{% if ansible_local|d() and ansible_local.gitlab_runner|d() and ansible_local.gitlab_runner.instance_tokens|d() %}
+{%   for item in ansible_local.gitlab_runner.instance_tokens %}
+{%     if item.name not in gitlab_runner__tpl_instances %}
+{%       for element in (gitlab_runner__default_instances + gitlab_runner__instances + gitlab_runner__group_instances + gitlab_runner__host_instances) %}
+{%         if (item.name == element.name and (element.state is undefined or element.state != 'absent')) %}
+{%           set _ = gitlab_runner__tpl_instances.append(item.name) %}
+{%           set _ = gitlab_runner__tpl_instance_tokens.append({ "name": item.name, "token": item.token }) %}
+{%         endif %}
+{%       endfor %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+{% set output = {
+    "instance_tokens": gitlab_runner__tpl_instance_tokens,
+    "instances": gitlab_runner__tpl_instances
+} %}
+{{ output | to_nice_json }}

templates/lookup/gitlab_runner__shell_tags.j2

@@ -0,0 +1,24 @@
+{% set output = [ 'shell' ] %}
+{% set _ = output.append(ansible_pkg_mgr) %}
+{% set _ = output.append(ansible_service_mgr) %}
+{% set _ = output.append(ansible_system | lower) %}
+{% if (ansible_local|d() and ansible_local.tags|d()) %}
+{%   set _ = output.extend(ansible_local.tags) %}
+{% endif %}
+{% if (ansible_local|d() and ansible_local.docker|d() and
+       (ansible_local.docker.installed|d()) | bool) %}
+{%   set _ = output.append('docker-host') %}
+{% endif %}
+{% set _ = output.append((ansible_local.core.distribution
+                          if (ansible_local|d() and ansible_local.core|d() and
+                              ansible_local.core.distribution|d())
+                          else ansible_distribution) | lower) %}
+{% set _ = output.append((ansible_local.core.distribution_release
+                          if (ansible_local|d() and ansible_local.core|d() and
+                              ansible_local.core.distribution_release|d())
+                          else ansible_distribution_release) | lower) %}
+{% if ansible_virtualization_type not in [ 'NA' ] %}
+{%   set _ = output.append(ansible_virtualization_type + '-' + ansible_virtualization_role) %}
+{% endif %}
+{% set _ = output.append('amd64' if (ansible_architecture == 'x86_64') else ansible_architecture) %}
+{{ output | unique }}