feat(ng): improve ng.clean state and ci (#300)

noelmcloughlin · web-flow · commit 92883e288cb9 · 2021-09-08T13:50:47.000+01:00
* feat(ng): improve ng.clean state and ci

* fix(clean): remove containers/compose before software
diff --git a/.github/workflows/kitchen.yml b/.github/workflows/kitchen.yml
@@ -38,7 +38,7 @@ jobs:
           # - package-debian-10-master-py3
           # - repo-debian-10-master-py3
           # - clean-debian-10-master-py3
-          - debian-10-master-py3
+          - dockeronly-debian-10-master-py3
           # - archive-debian-9-master-py3
           # - package-debian-9-master-py3
           # - repo-debian-9-master-py3
@@ -53,7 +53,7 @@ jobs:
           # - package-ubuntu-1804-master-py3
           # - repo-ubuntu-1804-master-py3
           # - clean-ubuntu-1804-master-py3
-          - ubuntu-1804-master-py3
+          - composeclean-ubuntu-1804-master-py3
           # - archive-centos-8-master-py3
           # - package-centos-8-master-py3
           # - repo-centos-8-master-py3
@@ -73,17 +73,17 @@ jobs:
           # - package-fedora-33-master-py3
           # - repo-fedora-33-master-py3
           # - clean-fedora-33-master-py3
-          - fedora-33-master-py3
+          - dockeronly-fedora-33-master-py3
           # - archive-opensuse-leap-153-master-py3
           # - package-opensuse-leap-153-master-py3
           # - repo-opensuse-leap-153-master-py3
           # - clean-opensuse-leap-153-master-py3
-          - opensuse-leap-153-master-py3
+          # opensuse-leap-153-master-py3
           # - archive-opensuse-leap-152-master-py3
           # - package-opensuse-leap-152-master-py3
           # - repo-opensuse-leap-152-master-py3
           # - clean-opensuse-leap-152-master-py3
-          - opensuse-leap-152-master-py3
+          # opensuse-leap-152-master-py3
           # - archive-opensuse-tmbl-latest-master-py3
           # - package-opensuse-tmbl-latest-master-py3
           # - repo-opensuse-tmbl-latest-master-py3
diff --git a/docker/clean.sls b/docker/clean.sls
@@ -2,6 +2,6 @@
 # vim: ft=sls
 
 include:
-  # .containers.clean
+  - .containers.clean
   - .compose.clean
   - .software.clean
diff --git a/docker/compose/clean.sls b/docker/compose/clean.sls
@@ -1,5 +1,26 @@
 # -*- coding: utf-8 -*-
 # vim: ft=sls
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import data as d with context %}
 
+{%- set sls_docker_software_clean = tplroot ~ '.software.clean' %}
+{%- set sls_compose_software_clean = tplroot ~ '.compose.software.clean' %}
 include:
-  - .software.clean
+  - {{ sls_compose_software_clean }}
+
+    {%- for name, container in d.compose.ng.items() %}
+
+docker-compose-ng-{{ container.container_name|d(name) }}-{{ loop.index }}-stopped:
+  docker_container.stopped:
+    - name: {{ container.container_name|d(name) }}
+    - onlyif: docker container inspect {{ container.container_name|d(name) }}
+    - require_in:
+      - sls: {{ sls_compose_software_clean }}
+
+docker-compose-ng-{{ container.image }}-{{ loop.index }}-absent:
+  docker_image.absent:
+    - name: {{ container.image }}
+    - require_in:
+      - sls: {{ sls_compose_software_clean }}
+
+     {% endfor %}
diff --git a/docker/compose/software/binary/clean.sls b/docker/compose/software/binary/clean.sls
@@ -14,13 +14,14 @@ docker-compose-archive-absent:
   file.absent:
     - names:
       - {{ d.dir.tmp }}/docker-compose
+        {%- if 'path' in d.pkg.compose %}
       - {{ d.pkg.compose.path }}
+        {%- endif %}
         {%- if d.linux.altpriority|int == 0 or grains.os_family in ('Arch', 'MacOS') %}
             {%- if 'commands' in d.pkg.compose %}
                 {%- for cmd in d.pkg.compose.commands|unique %}
       - /usr/local/bin/{{ cmd }}
                 {%- endfor %}
             {%- endif %}
         {%- endif %}
-
     {%- endif %}
diff --git a/docker/osfamilymap.yaml b/docker/osfamilymap.yaml
@@ -52,7 +52,6 @@ RedHat:
   pkg:
     deps:
       - python3-pip
-      # python3-docker
       - iptables
       - git
       - tar
@@ -67,8 +66,8 @@ RedHat:
 Suse:
   pkg:
     deps:
-      - python3-pip
-      - python3-docker
+      - python38-pip
+      - python38-docker
       - tar
       - gzip
     docker:
diff --git a/docker/osfingermap.yaml b/docker/osfingermap.yaml
@@ -31,6 +31,7 @@ CentOS Linux-7:
       - selinux-policy-minimum
       - yum-plugin-versionlock
       - python3-pip
+      - python36-docker
       - iptables
       - git
       - tar
diff --git a/docker/osmap.yaml b/docker/osmap.yaml
@@ -17,6 +17,7 @@ AlmaLinux:
   pkg:
     deps:
       - python3-dnf-plugin-versionlock
+      - python3-docker
     docker:
       repo:
         baseurl: 'https://download.docker.com/linux/centos/{{ grains.get('osmajorrelease', '') }}/$basearch/stable'
@@ -27,6 +28,7 @@ Rocky:
   pkg:
     deps:
       - python3-dnf-plugin-versionlock
+      - python3-docker
     docker:
       repo:
         baseurl: 'https://download.docker.com/linux/centos/{{ grains.get('osmajorrelease', '') }}/$basearch/stable'
@@ -68,3 +70,4 @@ Fedora:
     deps:
       - selinux-policy-minimum
       - python3-dnf-plugin-versionlock
+      - python3-docker
diff --git a/docker/software/archive/install.sls b/docker/software/archive/install.sls
@@ -10,7 +10,7 @@
 
 docker-software-docker-archive-install:
         {%- if 'deps' in d.pkg and d.pkg.deps %}
-            {%- if grains.os|lower in ('centos', 'redhat') %}
+            {%- if grains.os|lower in ('redhat', 'centos') %}
                 # python-docker package is not available or too old on CentOS, RedHat
                 # https://github.com/saltstack/salt/issues/58920
   pip.installed:
diff --git a/docker/software/package/clean.sls b/docker/software/package/clean.sls
@@ -15,7 +15,9 @@ include:
         {%- if grains.kernel|lower in ('linux', 'darwin') %}
 docker-software-package-clean-pkg:
   pkg.removed:
-    - name: {{ d.pkg.docker.name }}
+    - names:
+      - {{ d.pkg.docker.name }}
+      - python3-docker
     - reload_modules: {{ d.misc.reload|default(true, true) }}
             {%- if enable_repo %}
     - require:
diff --git a/kitchen.yml b/kitchen.yml
@@ -730,6 +730,7 @@ suites:
         base:
           '*':
             - docker._mapdata
+            - docker
             - docker.clean
       pillars:
         top.sls:
@@ -741,3 +742,38 @@ suites:
     verifier:
       inspec_tests:
         - path: test/integration/clean
+  - name: composeclean
+    provisioner:
+      state_top:
+        base:
+          '*':
+            - docker._mapdata
+            - docker
+            - docker.compose.clean
+      pillars:
+        top.sls:
+          base:
+            '*':
+              - docker
+      pillars_from_files:
+        docker.sls: test/salt/pillar/dockeronly.sls
+    verifier:
+      inspec_tests:
+        - path: test/integration/dockeronly
+  - name: dockeronly
+    provisioner:
+      state_top:
+        base:
+          '*':
+            - docker._mapdata
+            - docker
+      pillars:
+        top.sls:
+          base:
+            '*':
+              - docker
+      pillars_from_files:
+        docker.sls: test/salt/pillar/dockeronly.sls
+    verifier:
+      inspec_tests:
+        - path: test/integration/dockeronly
diff --git a/pillar.example b/pillar.example
@@ -19,7 +19,8 @@ docker:
         # yamllint enable-line rule:line-length
         - DOCKER_OPTS="-s btrfs --dns 8.8.8.8"
         - export http_proxy="http://172.17.42.1:3128"
-      daemon_config: {}
+      daemon_config:
+        insecure-registries: []
 
   networks:
     - nginxnet
diff --git a/test/integration/archive/controls/config.rb b/test/integration/archive/controls/config.rb
@@ -12,6 +12,10 @@
     its('content') { should include 'export http_proxy="http://172.17.42.1:3128"' }
   end
   describe file('/etc/docker/daemon.json') do
-    it { should_not exist }
+    it { should be_file }
+    its('owner') { should eq 'root' }
+    its('group') { should eq 'root' }
+    its('mode')  { should cmp '0640' }
+    its('content') { should include '"insecure-registries": []' }
   end
 end
diff --git a/test/integration/dockeronly/README.md b/test/integration/dockeronly/README.md
@@ -0,0 +1,50 @@
+# InSpec Profile: `dockeronly`
+
+This shows the implementation of the `dockeronly` InSpec [profile](https://github.com/inspec/inspec/blob/master/docs/profiles.md).
+
+## Verify a profile
+
+InSpec ships with built-in features to verify a profile structure.
+
+```bash
+$ inspec check dockeronly
+Summary
+-------
+Location: dockeronly
+Profile: profile
+Controls: 4
+Timestamp: 2019-06-24T23:09:01+00:00
+Valid: true
+
+Errors
+------
+
+Warnings
+--------
+```
+
+## Execute a profile
+
+To run all **supported** controls on a local machine use `inspec exec /path/to/profile`.
+
+```bash
+$ inspec exec dockeronly
+..
+
+Finished in 0.0025 seconds (files took 0.12449 seconds to load)
+8 examples, 0 failures
+```
+
+## Execute a specific control from a profile
+
+To run one control from the profile use `inspec exec /path/to/profile --controls name`.
+
+```bash
+$ inspec exec dockeronly --controls dockeronly
+.
+
+Finished in 0.0025 seconds (files took 0.12449 seconds to load)
+1 examples, 0 failures
+```
+
+See an [example control here](https://github.com/inspec/inspec/blob/master/examples/profile/controls/example.rb).
diff --git a/test/integration/dockeronly/controls/archive_spec.rb b/test/integration/dockeronly/controls/archive_spec.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+title 'docker archives profile'
+
+control 'docker archive' do
+  impact 1.0
+  title 'should be installed'
+
+  describe file('/usr/local/docker-19.03.9/bin') do
+    it { should exist }
+    it { should be_directory }
+    its('type') { should eq :directory }
+  end
+  describe file('/usr/local/docker-19.03.9/bin/docker') do
+    it { should exist }
+    its('mode') { should cmp '0755' }
+  end
+  describe file('/usr/local/docker-19.03.9/bin/runc') do
+    it { should exist }
+    its('mode') { should cmp '0755' }
+  end
+  describe file('/usr/local/docker-19.03.9/bin/docker-proxy') do
+    it { should exist }
+    its('mode') { should cmp '0755' }
+  end
+  describe file('/usr/local/docker-19.03.9/bin/containerd') do
+    it { should exist }
+    its('mode') { should cmp '0755' }
+  end
+  describe file('/usr/local/docker-19.03.9/bin/ctr') do
+    it { should exist }
+    its('mode') { should cmp '0755' }
+  end
+  describe file('/usr/local/docker-19.03.9/bin/dockerd') do
+    it { should exist }
+    its('mode') { should cmp '0755' }
+  end
+  describe file('/usr/local/docker-19.03.9/bin/containerd-shim') do
+    it { should exist }
+    its('mode') { should cmp '0755' }
+  end
+  describe file('/usr/local/bin/docker') do
+    it { should be_symlink }
+    it { should be_file }
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/bin/runc') do
+    it { should be_symlink }
+    it { should be_file }
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/bin/docker-proxy') do
+    it { should be_symlink }
+    it { should be_file }
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/bin/containerd') do
+    it { should be_symlink }
+    it { should be_file }
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/bin/ctr') do
+    it { should be_symlink }
+    it { should be_file }
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/bin/dockerd') do
+    it { should be_symlink }
+    it { should be_file }
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/bin/containerd-shim') do
+    it { should be_symlink }
+    it { should be_file }
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/docker-compose-latest/bin') do
+    it { should_not be_directory }
+  end
+  describe file('/usr/local/docker-compose-latest/bin/docker-compose') do
+    it { should_not be_file }
+  end
+  describe file('/usr/local/bin/docker-compose') do
+    it { should_not be_file }
+  end
+end
diff --git a/test/integration/dockeronly/controls/config.rb b/test/integration/dockeronly/controls/config.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+control 'Docker configuration' do
+  title 'should match desired lines'
+
+  describe file('/etc/default/docker') do
+    it { should be_file }
+    its('owner') { should eq 'root' }
+    its('group') { should eq 'root' }
+    its('mode')  { should cmp '0640' }
+    its('content') { should include 'DOCKER_OPTS="-s btrfs --dns 8.8.8.8"' }
+    its('content') { should include 'export http_proxy="http://172.17.42.1:3128"' }
+  end
+  describe file('/etc/docker/daemon.json') do
+    it { should be_file }
+    its('owner') { should eq 'root' }
+    its('group') { should eq 'root' }
+    its('mode')  { should cmp '0640' }
+    its('content') { should include '"insecure-registries": []' }
+  end
+end
diff --git a/test/integration/dockeronly/controls/service.rb b/test/integration/dockeronly/controls/service.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+control 'Docker service' do
+  title 'should be running and enabled'
+
+  describe service('docker') do
+    it { should be_installed }
+    it { should be_enabled }
+    it { should be_running } unless %w[fedora suse].include? platform[:family]
+  end
+end
diff --git a/test/integration/dockeronly/inspec.yml b/test/integration/dockeronly/inspec.yml
diff --git a/test/salt/pillar/dockeronly.sls b/test/salt/pillar/dockeronly.sls
