Merge pull request #274 from gerrod3/travisconf

Updating Travis config

Author: dkliban

.travis.yml

@@ -1,13 +1,13 @@
 # WARNING: DO NOT EDIT!
 #
-# This file was generated by plugin_template, and is managed by bootstrap.py. Please use
-# bootstrap.py to update this file.
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_python' to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
 ---
 sudo: required
 # https://docs.travis-ci.com/user/trusty-ci-environment/
-dist: xenial
+dist: bionic
 language: python
 python:
   # Fedora has 3.6 available (python36.x86_64), but pulp container images
@@ -18,11 +18,6 @@ env:
   matrix:
     - TEST=pulp
     - TEST=docs
-matrix:
-  exclude:
-    - python: '3.6'
-      env: TEST=docs
-  fast_finish: true
 services:
   - postgresql
   - redis-server
@@ -39,38 +34,39 @@ install: .travis/install.sh
 before_script: .travis/before_script.sh
 script: .travis/script.sh
 after_failure:
-  - http --timeout 30 --check-status --pretty format --print hb http://localhost:24817/pulp/api/v3/status/
-  - sudo docker images
-  - sudo kubectl logs -l name=pulp-operator -c ansible --tail=10000
-  - sudo kubectl logs -l name=pulp-operator -c operator --tail=10000
-  - sudo kubectl logs -l app=pulp-api --tail=50000
-  - sudo kubectl logs -l app=pulp-content --tail=10000
-  - sudo kubectl logs -l app=pulp-resource-manager --tail=10000
-  - sudo kubectl logs -l app=pulp-worker --tail=10000
+  - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/
+  - docker images
+  - docker ps -a
+  - docker logs pulp
 jobs:
   include:
-    - stage: deploy-plugin-to-pypi
+    - stage: deploy
+      name: "Deploy plugin to pypi"
       before_install: skip
       install: skip
       before_script: skip
       script: bash .travis/publish_plugin_pypi.sh
       if: tag IS present
-    - stage: publish-daily-client-gem
+    - stage: deploy
+      name: "Publish client to rubygems"
       script: bash .travis/publish_client_gem.sh
       env:
         - TEST=bindings
       if: type = cron
-    - stage: publish-daily-client-pypi
+    - stage: deploy
+      name: "Publish client to pypi"
       script: bash .travis/publish_client_pypi.sh
       env:
         - TEST=bindings
       if: type = cron
-    - stage: publish-client-gem
+    - stage: deploy
+      name: "Publish client to rubygems"
       script: bash .travis/publish_client_gem.sh
       env:
         - TEST=bindings
       if: tag IS present
-    - stage: publish-client-pypi
+    - stage: deploy
+      name: "Publish client to pypi"
       script: bash .travis/publish_client_pypi.sh
       env:
         - TEST=bindings

.travis/.gitignore

@@ -0,0 +1,3 @@
+Containerfile
+settings/
+vars/

.travis/Containerfile.j2

@@ -0,0 +1,23 @@
+FROM pulp/pulp-ci:latest
+
+{% if s3_test | default(false) %}
+# Hacking botocore (https://github.com/boto/botocore/pull/1990)
+RUN pip3 install django-storages[boto3]
+RUN pip3 install git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
+{% endif %}
+
+{% for item in plugins %}
+
+# Install {{ item.name }} from {{ item.source }}
+{% if item.name == "pulp-certguard" %}
+RUN pip3 install python-dateutil rhsm
+{% endif %}
+{% if item.source.startswith("./") %}
+ADD {{ item.source }} {{ item.source }}
+{% endif %}
+RUN pip3 install {{ item.source }}
+RUN mkdir -p /etc/nginx/pulp/
+RUN ln /usr/local/lib/python3.7/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
+{% endfor %}
+
+ENTRYPOINT ["/init"]

.travis/ansible.cfg

@@ -0,0 +1,7 @@
+[defaults]
+inventory = inventory.yaml
+filter_plugins = filter
+retry_files_enabled = False
+transport = local
+nocows = 1
+stdout_callback = yaml

.travis/before_install.sh

@@ -2,8 +2,8 @@
 
 # WARNING: DO NOT EDIT!
 #
-# This file was generated by plugin_template, and is managed by bootstrap.py. Please use
-# bootstrap.py to update this file.
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_python' to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
 
@@ -16,27 +16,23 @@ COMMIT_MSG=$(git log --format=%B --no-merges -1)
 export COMMIT_MSG
 
 if [ -f $PRE_BEFORE_INSTALL ]; then
-    $PRE_BEFORE_INSTALL
+  source $PRE_BEFORE_INSTALL
 fi
 
 if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*" | grep -v "https") ]]; then
   echo "Invalid Required PR link detected in commit message. Please use the full https url."
   exit 1
 fi
 
-if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ -z "$TRAVIS_TAG" -a "$TRAVIS_BRANCH" != "master"]
+if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ -z "$TRAVIS_TAG" -a "$TRAVIS_BRANCH" != "master" ]
 then
-  export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
-  export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
-  export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
-  export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
-  export PULP_OPERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-operator\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULPCORE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULP_OPENAPI_GENERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
 else
-  export PULP_PR_NUMBER=
+  export PULPCORE_PR_NUMBER=
   export PULP_SMASH_PR_NUMBER=
-  export PULP_ROLES_PR_NUMBER=
-  export PULP_BINDINGS_PR_NUMBER=
-  export PULP_OPERATOR_PR_NUMBER=
+  export PULP_OPENAPI_GENERATOR_PR_NUMBER=
 fi
 
 # dev_requirements contains tools needed for flake8, etc.
@@ -51,50 +47,45 @@ pip install -r dev_requirements.txt
 # Lint code.
 flake8 --config flake8.cfg
 
-cd ..
-git clone --depth=1 https://github.com/pulp/ansible-pulp.git
-if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
-  cd ansible-pulp
-  git fetch --depth=1 origin +refs/pull/$PULP_ROLES_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  cd ..
-fi
+# check for imports from pulpcore that aren't pulpcore.plugin
+./.travis/check_pulpcore_imports.sh
 
+cd ..
 
-git clone --depth=1 https://github.com/pulp/pulp-operator.git
-if [ -n "$PULP_OPERATOR_PR_NUMBER" ]; then
-  cd pulp-operator
-  git fetch --depth=1 origin +refs/pull/$PULP_OPERATOR_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  RELEASE_VERSION=v0.9.0
-  curl -LO https://github.com/operator-framework/operator-sdk/releases/download/${RELEASE_VERSION}/operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
-  chmod +x operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu && sudo mkdir -p /usr/local/bin/ && sudo cp operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk && rm operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
-  sudo operator-sdk build --image-builder=docker quay.io/pulp/pulp-operator:latest
+git clone https://github.com/pulp/pulp-openapi-generator.git
+if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then
+  cd pulp-openapi-generator
+  git fetch origin pull/$PULP_OPENAPI_GENERATOR_PR_NUMBER/head:$PULP_OPENAPI_GENERATOR_PR_NUMBER
+  git checkout $PULP_OPENAPI_GENERATOR_PR_NUMBER
   cd ..
 fi
 
+cd pulp-openapi-generator
+sed -i -e 's/localhost:24817/pulp/g' generate.sh
+sed -i -e 's/:24817/pulp/g' generate.sh
+cd ..
 
-git clone --depth=1 https://github.com/pulp/pulpcore.git
+git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master
 
-if [ -n "$PULP_PR_NUMBER" ]; then
-  cd pulpcore
-  git fetch --depth=1 origin +refs/pull/$PULP_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  cd ..
+cd pulpcore
+if [ -n "$PULPCORE_PR_NUMBER" ]; then
+  git fetch --depth=1 origin pull/$PULPCORE_PR_NUMBER/head:$PULPCORE_PR_NUMBER
+  git checkout $PULPCORE_PR_NUMBER
 fi
+cd ..
 
 
 
 # When building a (release) tag, we don't need the development modules for the
 # build (they will be installed as dependencies of the plugin).
 if [ -z "$TRAVIS_TAG" ]; then
 
-  git clone --depth=1 https://github.com/PulpQE/pulp-smash.git
+  git clone --depth=1 https://github.com/pulp/pulp-smash.git
 
   if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
     cd pulp-smash
-    git fetch --depth=1 origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
-    git checkout FETCH_HEAD
+    git fetch --depth=1 origin pull/$PULP_SMASH_PR_NUMBER/head:$PULP_SMASH_PR_NUMBER
+    git checkout $PULP_SMASH_PR_NUMBER
     cd ..
   fi
 
@@ -104,10 +95,17 @@ if [ -z "$TRAVIS_TAG" ]; then
 fi
 
 
-pip install ansible
+# Intall requirements for ansible playbooks
+pip install docker netaddr boto3
+
+# Install ansible with the boto3 tags to dict fix
+# There is a PR for this issue:
+# https://github.com/ansible-collections/amazon.aws/pull/37
+# Be aware, that the code will have moved to that collection with upcoming releases of ansible
+pip install git+https://github.com/mdellweg/ansible.git@fix_boto3_tags_dict
 
 cd pulp_python
 
 if [ -f $POST_BEFORE_INSTALL ]; then
-    $POST_BEFORE_INSTALL
+  source $POST_BEFORE_INSTALL
 fi

.travis/before_script.sh

@@ -2,44 +2,31 @@
 
 # WARNING: DO NOT EDIT!
 #
-# This file was generated by plugin_template, and is managed by bootstrap.py. Please use
-# bootstrap.py to update this file.
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_python' to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
 
 set -euv
 
+source .travis/utils.sh
+
 export PRE_BEFORE_SCRIPT=$TRAVIS_BUILD_DIR/.travis/pre_before_script.sh
 export POST_BEFORE_SCRIPT=$TRAVIS_BUILD_DIR/.travis/post_before_script.sh
 
-# Aliases for running commands in the pulp-api container.
-export PULP_API_POD=$(sudo kubectl get pods | grep -E -o "pulp-api-(\w+)-(\w+)")
-# Run a command
-export CMD_PREFIX="sudo kubectl exec $PULP_API_POD --"
-# Run a command, and pass STDIN
-export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --"
-
 if [[ -f $PRE_BEFORE_SCRIPT ]]; then
-    $PRE_BEFORE_SCRIPT
+  source $PRE_BEFORE_SCRIPT
 fi
 
-mkdir -p ~/.config/pulp_smash
-
-if [[ -f .travis/pulp-smash-config.json ]]; then
-    sed "s/localhost/$(hostname)/g" .travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
-else
-    sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
-fi
+# Developers often want to know the final pulp config
+echo "PULP CONFIG:"
+tail -v -n +1 .travis/settings/settings.* ~/.config/pulp_smash/settings.json
 
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then
-    # Many tests require pytest/mock, but users do not need them at runtime
-    # (or to add plugins on top of pulpcore or pulp container images.)
-    # So install it here, rather than in the image Dockerfile.
-    $CMD_PREFIX pip3 install pytest mock
-    # Many functional tests require these
-    $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core
+if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' ]]; then
+  # Many functional tests require these
+  cmd_prefix dnf install -yq lsof which dnf-plugins-core
 fi
 
 if [[ -f $POST_BEFORE_SCRIPT ]]; then
-    $POST_BEFORE_SCRIPT
+  source $POST_BEFORE_SCRIPT
 fi

.travis/build_container.yaml

@@ -0,0 +1,36 @@
+# Ansible playbook to create the pulp service containers image
+---
+- hosts: localhost
+  gather_facts: false
+  vars_files:
+    - vars/main.yaml
+  tasks:
+    - name: "Generate Containerfile from template"
+      template:
+        src: Containerfile.j2
+        dest: Containerfile
+
+    - name: "Build pulp image"
+      docker_image:
+      # We build from the ../.. (parent dir of pulpcore git repo) Docker build
+      # "context" so that repos like pulp-smash are accessible to Docker
+      # build. So that PR branches can be used via relative paths.
+      #
+      # We default to using the docker build / podman buildah cache, for
+      # 1-off-builds and Travis CI purposes (which has no cache across CI runs.)
+      # Run build.yaml with -e cache=false if your builds are using outdated
+      # layers.
+        name: "{{ image.name }}"
+        tag: "{{ image.tag }}"
+        build:
+          path: "../.."
+          dockerfile: "{{ playbook_dir }}/Containerfile"
+          nocache: "{{ not cache | default(true) | bool }}"
+          pull: false
+        state: present
+        source: build
+
+    - name: "Clean image cache"
+      docker_prune:
+        images : true
+...

.travis/check_commit.sh

@@ -2,8 +2,8 @@
 
 # WARNING: DO NOT EDIT!
 #
-# This file was generated by plugin_template, and is managed by bootstrap.py. Please use
-# bootstrap.py to update this file.
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_python' to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
 

.travis/check_pulpcore_imports.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_python' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+set -uv
+
+# check for imports not from pulpcore.plugin. exclude tests
+MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin")
+
+if [ $? -ne 1 ]; then
+  printf "\nERROR: Detected bad imports from pulpcore:\n"
+  echo "$MATCHES"
+  printf "\nPlugins should import from pulpcore.plugin."
+  exit 1
+fi