From b091e06ad6f96f1bb98be9be1810ee65ea14da3a Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Mon, 18 Nov 2019 18:41:59 -0800 Subject: [PATCH 1/8] Check licenses for compatibility and reject licenses not in whitelist --- .travis.yml | 2 + Makefile | 4 + config/license_finder.yml | 4 + config/license_finder_decisions.yml | 235 ++++++++++++++++++++++++++++ requirements.txt | 4 +- 5 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 config/license_finder.yml create mode 100644 config/license_finder_decisions.yml diff --git a/.travis.yml b/.travis.yml index 937cf73f..4dbb4ae6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,13 @@ services: before_install: - docker build -f Dockerfile -t $REPO:$TRAVIS_COMMIT . - docker run -v $PWD/confidant/dist:/tmp/dist $REPO:$TRAVIS_COMMIT /bin/sh -c "cp -r /srv/confidant/confidant/dist/. /tmp/dist/." + - docker pull licensefinder/license_finder install: - gem install travis --no-document script: - travis lint .travis.yml --skip-completion-check - docker run $REPO:$TRAVIS_COMMIT /bin/sh -c "make test" + - docker run -v $PWD/confidant:/scan -it licensefinder/license_finder "make test_licenses" after_success: - ./docker_push.sh deploy: diff --git a/Makefile b/Makefile index 55c47b75..13da5a1d 100644 --- a/Makefile +++ b/Makefile @@ -9,3 +9,7 @@ test_lint: test_unit: nosetests --with-path=confidant tests/unit + +test_licenses: + apt-get install -y build-essential libffi-dev libxml2-dev libxmlsec1-dev python-dev + license_finder -p diff --git a/config/license_finder.yml b/config/license_finder.yml new file mode 100644 index 00000000..02cdf139 --- /dev/null +++ b/config/license_finder.yml @@ -0,0 +1,4 @@ +--- +decisions_file: './config/license_finder_decisions.yml' +python_version: '3' +pip_requirements_path: './requirements3.txt' diff --git a/config/license_finder_decisions.yml b/config/license_finder_decisions.yml new file mode 100644 index 00000000..71c42235 --- /dev/null +++ b/config/license_finder_decisions.yml @@ -0,0 +1,235 @@ +--- +- - :whitelist + - MIT + - :who: + :why: + :versions: [] + :when: 2019-11-19 01:55:51.429439100 Z +- - :whitelist + - ISC + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:00:10.960824500 Z +- - :whitelist + - BSD + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:00:48.608694400 Z +- - :whitelist + - Python Software Foundation License + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:01:14.298710000 Z +- - :whitelist + - Simplified BSD + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:01:28.972522900 Z +- - :whitelist + - Apache 2.0 + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:01:42.801093000 Z +- - :whitelist + - CC0-1.0 + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:01:57.707287400 Z +- - :whitelist + - MIT/X11 + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:02:08.599410600 Z +- - :whitelist + - New BSD + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:02:20.566269000 Z +- - :whitelist + - Expat license + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:04:52.898901000 Z +- - :whitelist + - ASL + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:06:33.891629700 Z +- - :whitelist + - Mozilla Public License 2.0 + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:06:56.556790400 Z +- - :whitelist + - LGPL + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:15:55.862389300 Z +- - :whitelist + - Apache License v2.0 + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:16:05.679307500 Z +- - :whitelist + - Public Domain + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:16:14.403847400 Z +- - :whitelist + - GNU LGPL + - :who: + :why: + :versions: [] + :when: 2019-11-19 02:16:20.654080000 Z +- - :approve + - active-x-obfuscator + - :who: + :why: Uses MIT + :versions: [] + :when: 2019-11-19 02:17:44.734536300 Z +- - :approve + - amdefine + - :who: + :why: Dual licensed, with two approved licenses + :versions: [] + :when: 2019-11-19 02:23:04.653198000 Z +- - :approve + - atob + - :who: + :why: Dual licensed, with two approved licenses + :versions: [] + :when: 2019-11-19 02:23:13.345680600 Z +- - :approve + - aws-sign2 + - :who: + :why: MIT licensed, see js file header + :versions: [] + :when: 2019-11-19 02:23:21.677034600 Z +- - :approve + - base64id + - :who: + :why: MIT licensed + :versions: [] + :when: 2019-11-19 02:23:30.465542700 Z +- - :approve + - buffers + - :who: + :why: MIT licensed + :versions: [] + :when: 2019-11-19 02:23:39.743678500 Z +- - :approve + - cryptography + - :who: + :why: Dual licensed, with two approved licenses + :versions: [] + :when: 2019-11-19 02:23:51.463313600 Z +- - :approve + - dateformat + - :who: + :why: MIT license + :versions: [] + :when: 2019-11-19 02:24:34.761542300 Z +- - :approve + - docutils + - :who: + :why: used for doc generation, not as library, using PD part + :versions: [] + :when: 2019-11-19 02:28:22.199427300 Z +- - :approve + - domhandler + - :who: + :why: BSD2 licensed + :versions: [] + :when: 2019-11-19 02:28:53.758951800 Z +- - :approve + - domutils + - :who: + :why: BSD2 licensed + :versions: [] + :when: 2019-11-19 02:29:08.209368400 Z +- - :approve + - entities + - :who: + :why: Modified BSD, checked and valid + :versions: [] + :when: 2019-11-19 02:29:32.168686800 Z +- - :approve + - forever-agent + - :who: + :why: Apache2 licensed + :versions: [] + :when: 2019-11-19 02:30:07.156268000 Z +- - :approve + - idna + - :who: + :why: Modified BSD, checked and valid + :versions: [] + :when: 2019-11-19 02:30:21.115549400 Z +- - :approve + - ndg-httpsclient + - :who: + :why: BSD licensed + :versions: [] + :when: 2019-11-19 02:30:38.728308800 Z +- - :approve + - oauth-sign + - :who: + :why: Apache2 licensed + :versions: [] + :when: 2019-11-19 02:31:01.569389300 Z +- - :approve + - python-dateutil + - :who: + :why: BSD and Apache2 licensed + :versions: [] + :when: 2019-11-19 02:31:30.792815000 Z +- - :approve + - rc + - :who: + :why: Multi-licensed under approved licenses + :versions: [] + :when: 2019-11-19 02:32:08.459004400 Z +- - :approve + - shelljs + - :who: + :why: BSD licensed + :versions: [] + :when: 2019-11-19 02:32:27.739563800 Z +- - :approve + - spdx-exceptions + - :who: + :why: Only data, so CC-BY-3.0 is OK + :versions: [] + :when: 2019-11-19 02:32:47.869039500 Z +- - :approve + - tinycolor + - :who: + :why: MIT licensed + :versions: [] + :when: 2019-11-19 02:33:15.853370100 Z +- - :approve + - tweetnacl + - :who: + :why: Effectively PD license, sigh + :versions: [] + :when: 2019-11-19 02:33:52.145372300 Z +- - :approve + - uglify-js + - :who: + :why: BSD license in README + :versions: [] + :when: 2019-11-19 02:34:38.062177700 Z diff --git a/requirements.txt b/requirements.txt index edfdd57b..7207ef94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,5 +63,5 @@ urllib3==1.25.3 # via botocore, requests werkzeug==0.15.6 # via flask xmlsec==1.3.3 # via python3-saml -pip==9.0.3 -setuptools==39.0.1 +#pip==9.0.3 +#setuptools==39.0.1 From 26973f77432224aaa9107c83ad764ec65e8d4a0b Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Mon, 18 Nov 2019 18:42:50 -0800 Subject: [PATCH 2/8] Uncomment some reqs --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7207ef94..edfdd57b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,5 +63,5 @@ urllib3==1.25.3 # via botocore, requests werkzeug==0.15.6 # via flask xmlsec==1.3.3 # via python3-saml -#pip==9.0.3 -#setuptools==39.0.1 +pip==9.0.3 +setuptools==39.0.1 From c0076074c38e8b411ca98c51a80b3df17d08ae6e Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Mon, 18 Nov 2019 19:41:08 -0800 Subject: [PATCH 3/8] Fix travis file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4dbb4ae6..cb7b92f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ install: script: - travis lint .travis.yml --skip-completion-check - docker run $REPO:$TRAVIS_COMMIT /bin/sh -c "make test" - - docker run -v $PWD/confidant:/scan -it licensefinder/license_finder "make test_licenses" + - docker run -v $PWD:/scan -it licensefinder/license_finder /bin/bash -lc "cd /scan && make test_licenses" after_success: - ./docker_push.sh deploy: From d4ac8e8b651ad195f537ce9677c2aae59003a97d Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 19 Nov 2019 15:27:00 -0800 Subject: [PATCH 4/8] use a matrix build --- .travis.yml | 31 ++++++++++++++++++++----------- Makefile | 12 ++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb7b92f0..bacce476 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,29 @@ dist: bionic language: python -env: - - REPO=lyft/confidant -sudo: required -services: - - docker -before_install: - - docker build -f Dockerfile -t $REPO:$TRAVIS_COMMIT . - - docker run -v $PWD/confidant/dist:/tmp/dist $REPO:$TRAVIS_COMMIT /bin/sh -c "cp -r /srv/confidant/confidant/dist/. /tmp/dist/." - - docker pull licensefinder/license_finder +matrix: + include: + - env: + - 'REPO=lyft/confidant' + - 'TEST_SUITE=docker' + sudo: required + services: + - docker + before_install: + - docker build -f Dockerfile -t $REPO:$TRAVIS_COMMIT . + - docker run -v $PWD/confidant/dist:/tmp/dist $REPO:$TRAVIS_COMMIT /bin/sh -c "cp -r /srv/confidant/confidant/dist/. /tmp/dist/." + - env: + - 'REPO=lyft/confidant' + - 'TEST_SUITE=licenses_docker' + sudo: required + services: + - docker + before_install: + - docker pull licensefinder/license_finder install: - gem install travis --no-document script: - travis lint .travis.yml --skip-completion-check - - docker run $REPO:$TRAVIS_COMMIT /bin/sh -c "make test" - - docker run -v $PWD:/scan -it licensefinder/license_finder /bin/bash -lc "cd /scan && make test_licenses" + - docker run $REPO:$TRAVIS_COMMIT /bin/sh -c "make test${TEST_SUITE}" after_success: - ./docker_push.sh deploy: diff --git a/Makefile b/Makefile index 13da5a1d..d68f7306 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,25 @@ SHELL := /bin/bash test: test_lint test_unit +test_docker: + docker run ${REPO}:${TRAVIS_COMMIT} /bin/sh -c "make test" + test_lint: mkdir -p build set -o pipefail; flake8 | sed "s#^\./##" > build/flake8.txt || (cat build/flake8.txt && exit 1) +test_lint_docker: + docker run ${REPO}:${TRAVIS_COMMIT} /bin/sh -c "make test_lint" + test_unit: nosetests --with-path=confidant tests/unit +test_unit_docker: + docker run ${REPO}:${TRAVIS_COMMIT} /bin/sh -c "make test_unit" + test_licenses: apt-get install -y build-essential libffi-dev libxml2-dev libxmlsec1-dev python-dev license_finder -p + +test_licenses_docker: + docker run -v $PWD:/scan -it licensefinder/license_finder /bin/bash -lc "cd /scan && make test_licenses" From 1bb53e40fd991bf13b15e485993f1e2ccaf91d36 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 19 Nov 2019 15:37:09 -0800 Subject: [PATCH 5/8] Fixes --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bacce476..8e88ba66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,8 @@ matrix: - docker before_install: - docker pull licensefinder/license_finder -install: - - gem install travis --no-document script: - - travis lint .travis.yml --skip-completion-check - - docker run $REPO:$TRAVIS_COMMIT /bin/sh -c "make test${TEST_SUITE}" + - make test_${TEST_SUITE} after_success: - ./docker_push.sh deploy: From d1781598672bdb3dbaa0afda44dd384485902853 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 19 Nov 2019 15:51:08 -0800 Subject: [PATCH 6/8] Use generic language, to avoid install step --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e88ba66..7f2a7fdb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ dist: bionic -language: python +language: generic matrix: include: - env: @@ -12,7 +12,6 @@ matrix: - docker build -f Dockerfile -t $REPO:$TRAVIS_COMMIT . - docker run -v $PWD/confidant/dist:/tmp/dist $REPO:$TRAVIS_COMMIT /bin/sh -c "cp -r /srv/confidant/confidant/dist/. /tmp/dist/." - env: - - 'REPO=lyft/confidant' - 'TEST_SUITE=licenses_docker' sudo: required services: From ebcc757cdbcb018e84a59e58d7e7a782ab0fbcc8 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 19 Nov 2019 16:16:49 -0800 Subject: [PATCH 7/8] No install step --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7f2a7fdb..c295228d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ matrix: - docker before_install: - docker pull licensefinder/license_finder +install: + - echo "no install step" script: - make test_${TEST_SUITE} after_success: From 16b0325ba42670cbc4f0ddf0bf36647ebe5494d0 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 19 Nov 2019 16:28:14 -0800 Subject: [PATCH 8/8] Wrap env --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d68f7306..f41d5cbd 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,4 @@ test_licenses: license_finder -p test_licenses_docker: - docker run -v $PWD:/scan -it licensefinder/license_finder /bin/bash -lc "cd /scan && make test_licenses" + docker run -v ${PWD}:/scan -it licensefinder/license_finder /bin/bash -lc "cd /scan && make test_licenses"