Fix CI configuration (for real this time..)

Author: jbaiter

appveyor.yml

@@ -20,32 +20,25 @@ environment:
       RUST_TARGET: "x86_64-pc-windows-msvc"
 
 install:
-    # Install Python dependencies
-  - "%PYTHON%\\python.exe -m pip install -U pip setuptools distribute"
-  - "%PYTHON%\\python.exe -m pip install -U wheel cffi pytest"
-    # Install Rust
+  - ".build.cmd %PYTHON%\\python -m pip install -U pip setuptools wheel"
   - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-nightly-${env:RUST_TARGET}.exe"
   - rust-nightly-%RUST_TARGET%.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust"
   - SET PATH=%PATH%;C:\Program Files (x86)\Rust\bin
   - SET PATH=%PATH%;C:\MinGW\bin
   - rustc -V
   - cargo -V
-  - cd c:\projects\python-rust-fst
-    # For some reason the binary wheel only gets built the second time we run this...
-  - ".build.cmd %PYTHON%\\python setup.py bdist_wheel || echo 'First build step done...'"
-    # Remove the bad wheel
-  - del /s /q /f dist\*any.whl
-    # Build the right wheel
-  - ".build.cmd %PYTHON%\\python setup.py bdist_wheel"
 
-build: off
+build_script:
+  - cd c:\projects\python-rust-fst
+  - ".build.cmd %PYTHON%\\python -m pip -v wheel . -w .\\wheelhouse"
+  - dir wheelhouse
 
 test_script:
-    # We have to be outside of the project directory for the tests
   - SET PROJPATH=C:\Projects\python-rust-fst
   - cd c:\projects
-  - "%PROJPATH%\\.build.cmd %PYTHON%\\python -m pip install rust_fst --no-index -f %PROJPATH%\\dist"
-  - "%PROJPATH%\\.build.cmd %PYTHON%\\python -m pytest %PROJPATH%\\tests"
+  - "%PROJPATH%\\.build.cmd %PYTHON%\\python -m pip install cffi pytest"
+  - "%PROJPATH%\\.build.cmd %PYTHON%\\python -m pip -v install rust_fst --no-index -f %PROJPATH%\\wheelhouse"
+  - "%PROJPATH%\\.build.cmd %PYTHON%\\python -m pytest --verbose %PROJPATH%\\tests"
 
 artifacts:
-  - path: dist\*
+  - path: wheelhouse\rust_fst*.whl

build-wheels.sh

@@ -1,68 +1,87 @@
 #!/bin/bash
 set -e -x
 
+function install_openssl {
+    # Compile and parallel-install a newer OpenSSL version so that curl can
+    # download from the rust servers
+    pushd /usr/src
+    wget -q ftp://ftp.openssl.org/source/openssl-${1}.tar.gz
+    tar xf openssl-${1}.tar.gz
+    cd openssl-${1}
+    ./config --prefix=/opt/openssl shared > /dev/null
+    make > /dev/null
+    make install > /dev/null
+    export LD_LIBRARY_PATH=/opt/openssl/lib:$LD_LIBRARY_PATH
+    popd
+}
+
+function install_curl {
+    pushd /usr/src
+    # Compile an up-to-date curl version that links to our own OpenSSL installation
+    wget -q --no-check-certificate http://curl.haxx.se/download/curl-${1}.tar.gz
+    tar xf curl-${1}.tar.gz
+    cd curl-${1}
+    ./configure --with-ssl=/opt/openssl --prefix=/opt/curl > /dev/null
+    make > /dev/null
+    make install > /dev/null
+    export PATH=/opt/curl/bin:$PATH
+    popd
+}
+
+function install_rust {
+    curl https://static.rust-lang.org/rustup.sh > /tmp/rustup.sh
+    chmod +x /tmp/rustup.sh
+    /tmp/rustup.sh -y --disable-sudo --channel=$1
+}
+
+function update_certificates {
+    # Update the Root CA bundle
+    wget -q --no-check-certificate \
+        -O /etc/pki/tls/certs/ca-bundle.crt \
+        http://curl.haxx.se/ca/cacert.pem
+}
+
+function clean_project {
+    # Remove compiled files that might cause conflicts
+    pushd /io/
+    rm -rf .cache .eggs rust_fst/_ffi.py build *.egg-info
+    find ./ -name "__pycache__" -type d -print0 |xargs rm -rf --
+    find ./ -name "*.pyc" -type f -print0 |xargs rm -rf --
+    popd
+}
+
 OPENSSL_VERSION=1.0.2h
 CURL_VERSION=7.49.0
 RUST_CHANNEL=nightly
 
+# Clean build files
+clean_project
+
+install_openssl $OPENSSL_VERSION
+install_curl $CURL_VERSION
+install_rust $RUST_CHANNEL
+
 # Remove old wheels
 rm -rf /io/wheelhouse/* || echo "No old wheels to delete"
 
 # We don't support Python 2.6
 rm -rf /opt/python/cp26*
 
-# Override PATH and LD_LIBRARY so that our curl and openssl installations
-# get precedence over the included versions
-PATH=/opt/curl/bin:$PATH
-LD_LIBRARY_PATH=/opt/openssl/lib:$LD_LIBRARY_PATH
-
 # Install libraries needed for compiling the extension
-yum -y install libffi-devel
-
-# Update the Root CA bundle
-wget --no-check-certificate \
-    -O /etc/pki/tls/certs/ca-bundle.crt \
-    http://curl.haxx.se/ca/cacert.pem
-
-# Compile and parallel-install a newer OpenSSL version so that curl can
-# download from the rust servers
-# Note that we cannot download from the official OpenSSL servers, since they
-# require HTTPS, which does not work with the CentOS5 SSL version...
-cd /usr/src
-wget http://ftp.vim.org/security/openssl/openssl-${OPENSSL_VERSION}.tar.gz
-tar xf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION}
-./config --prefix=/opt/openssl shared
-make
-make install
-cd ..
-
-# Compile an up-to-date curl version that links to our own OpenSSL installation
-wget --no-check-certificate http://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
-tar xf curl-${CURL_VERSION}.tar.gz
-cd curl-${CURL_VERSION}
-./configure --with-ssl=/opt/openssl --prefix=/opt/curl
-make
-make install
-cd $HOME
-
-# Install rust
-curl https://static.rust-lang.org/rustup.sh > /tmp/rustup.sh
-chmod +x /tmp/rustup.sh
-/tmp/rustup.sh -y --disable-sudo --channel=$RUST_CHANNEL
+yum -q -y install libffi-devel
 
 # Compile wheels
 for PYBIN in /opt/python/*/bin; do
-    ${PYBIN}/python -m pip install pytest
-    ${PYBIN}/python -m pip  wheel /io/ -w wheelhouse/
+    ${PYBIN}/python -m pip wheel /io/ -w /wheelhouse/
+    clean_project
 done
 
 # Move pure wheels to output wheelhouse
 mkdir -p /io/wheelhouse/
-mv wheelhouse/*any.whl /io/wheelhouse/ || echo "No pure wheels found."
+mv /wheelhouse/*any.whl /io/wheelhouse/ || echo "No pure wheels found."
 
 # Bundle external shared libraries into the wheels
-for whl in wheelhouse/*.whl; do
+for whl in /wheelhouse/*.whl; do
     auditwheel repair $whl -w /io/wheelhouse/
 done
 
@@ -71,9 +90,8 @@ chmod -R a+rw /io/wheelhouse
 
 # Install packages and test
 for PYBIN in /opt/python/*/bin/; do
+    ${PYBIN}/python -m pip install pytest
     ${PYBIN}/python -m pip install rust_fst --no-index -f /io/wheelhouse
     ${PYBIN}/python -m pytest --verbose /io/tests
-    rm -f /io/rust_fst/_ffi.py
-    find /io -name "__pycache__" -type d -print0 |xargs rm -rf --
-    find /io -name "*.pyc" -type f -print0 |xargs rm -rf --
+    clean_project
 done

rust_setuptools.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
-""" Code by Armin Ronacher.
+""" Based on code by Armin Ronacher and James Salter
 
 https://github.com/mitsuhiko/rust-setuptools
+https://github.com/novocaine/rust-python-ext
 """
 from __future__ import print_function
 
@@ -12,6 +13,7 @@
 
 from distutils.cmd import Command
 from distutils.command.install_lib import install_lib
+from distutils.dist import Distribution
 
 
 if sys.platform == 'win32':
@@ -22,6 +24,16 @@
     DYNAMIC_LIB_SUFFIX = '.so'
 
 
+class RustDistribution(Distribution):
+
+    def __init__(self, attrs=None):
+        Distribution.__init__(self, attrs)
+        self.ext_modules = []
+
+    def has_ext_modules(self):
+        return True
+
+
 class RustBuildCommand(Command):
     description = 'build rust crates into Python extensions'
 
@@ -35,6 +47,10 @@ def finalize_options(self):
         pass
 
     def run(self):
+        # Force binary wheel
+        self.distribution.has_ext_modules = lambda: True
+        self.distribution.ext_modules = []
+
         # Make sure that if pythonXX-sys is used, it builds against the
         # current executing python interpreter.
         bindir = os.path.dirname(sys.executable)
@@ -111,7 +127,6 @@ class _RustBuildCommand(RustBuildCommand):
             'extra_cargo_args': extra_cargo_args,
             'quiet': quiet,
         }
-
     return _RustBuildCommand
 
 
@@ -124,11 +139,3 @@ def build(self):
             if not self.skip_build:
                 self.run_command('build_rust')
     return _RustInstallLibCommand
-
-
-def rust_crates(dist, attr, value):
-    dist.has_ext_modules = lambda: True
-    dist.ext_modules = []
-    dist.cmdclass['build_rust'] = build_rust_cmdclass(value)
-    dist.cmdclass['install_lib'] = build_install_lib_cmdclass(
-        dist.cmdclass.get('install_lib'))

setup.py

@@ -1,5 +1,7 @@
 from setuptools import setup
 
+from rust_setuptools import (build_rust_cmdclass, build_install_lib_cmdclass,
+                             RustDistribution)
 
 setup(
     name='rust-fst',
@@ -17,11 +19,12 @@
         'pytest-runner'],
     install_requires=['cffi>=1.0.0'],
     cffi_modules=['rust_fst/_build_ffi.py:ffi'],
-    entry_points={'distutils.setup_keywords': [
-        'rust_crates = rust_setuptools:rust_crates'
-    ]},
     tests_require=['pytest'],
-    rust_crates=[('fstwrapper', 'rust_fst')],
+    distclass=RustDistribution,
+    cmdclass={
+        'build_rust': build_rust_cmdclass([('fstwrapper', 'rust_fst')]),
+        'install_lib': build_install_lib_cmdclass()
+    },
     packages=['rust_fst'],
     zip_safe=False,
     classifiers=[