fix the issue that CPU&MEM keeps increasing overtime

Author: tifayuki

Dockerfile

@@ -1,36 +1,16 @@
 FROM alpine:edge
 MAINTAINER Feng Honglin <hfeng@tutum.co>
 
-# Install tini, haproxy, pip and the dockercloud-haproxy python package:
-RUN apk --no-cache add \
-    tini \
-    haproxy \
-    py-pip \
-  && apk --no-cache add --virtual deps git \
-  && pip install --upgrade \
-    pip \
-  && apk del deps \
-  # Clean up obsolete files:
-  && rm -rf \
-    # Clean up any temporary files:
-    /tmp/* \
-    # Clean up the pip cache:
-    /root/.cache \
-    # Remove any compiled python files (compile on demand):
-    `find / -regex '.*\.py[co]'`
+COPY . /haproxy-src
 
-COPY reload.sh /reload.sh
-COPY . haproxy-src/
-RUN cd /haproxy-src/ && \
-  pip install . \
-  # Clean up obsolete files:
-  && rm -rf \
-    # Clean up any temporary files:
-    /tmp/* \
-    # Clean up the pip cache:
-    /root/.cache \
-    # Remove any compiled python files (compile on demand):
-    `find / -regex '.*\.py[co]'`
+RUN apk update && \
+    apk --no-cache add tini haproxy py-pip build-base python-dev ca-certificates && \
+    cp /haproxy-src/reload.sh /reload.sh && \
+    cd /haproxy-src && \
+    pip install -r requirements.txt && \
+    pip install . && \
+    apk del build-base python-dev && \
+    rm -rf "/tmp/*" "/root/.cache" `find / -regex '.*\.py[co]'`
 
 ENV RSYSLOG_DESTINATION=127.0.0.1 \
     MODE=http \

haproxy/helper/cloud_link_helper.py

@@ -1,10 +1,12 @@
-from multiprocessing.pool import ThreadPool
+from gevent.pool import Pool
 
 from haproxy.config import SERVICE_NAME_MATCH
 from haproxy.utils import get_uuid_from_resource_uri, fetch_remote_obj
 
 LINKED_CONTAINER_CACHE = {}
 
+pool = Pool(size=5)
+
 
 def get_cloud_links(haproxy_container):
     links = _init_links(haproxy_container.linked_to_container)
@@ -41,7 +43,7 @@ def _get_new_added_link_uri(container_object_cache, links):
 
 
 def _get_container_object_from_uri(container_uris):
-    pool = ThreadPool(processes=10)
+    global pool
     container_objects = pool.map(fetch_remote_obj, container_uris)
     return container_objects
 

haproxy/main.py

@@ -13,6 +13,10 @@
 from haproxycfg import run_haproxy
 from utils import save_to_file
 
+
+from gevent import monkey
+monkey.patch_all()
+
 dockercloud.user_agent = "dockercloud-haproxy/%s" % __version__
 
 logger = logging.getLogger("haproxy")

requirements.txt

@@ -1,2 +1,3 @@
 python-dockercloud==1.0.3
 docker-compose==1.6.0
+gevent==1.1.1

setup.py

@@ -4,6 +4,11 @@
 
 from setuptools import setup, find_packages
 
+requirements = [
+    "python-dockercloud >= 1.0.3, < 2",
+    "docker-compose >= 1.6.0, <2"
+    ]
+
 
 def read(*parts):
     path = os.path.join(os.path.dirname(__file__), *parts)
@@ -20,17 +25,14 @@ def find_version(*file_paths):
     raise RuntimeError('Unable to find version string.')
 
 
-with open('requirements.txt') as f:
-    install_requires = f.read().splitlines()
-
 with open('./test-requirements.txt') as test_reqs_txt:
     test_requirements = [line for line in test_reqs_txt]
 
 setup(
     name='dockercloud-haproxy',
     version=find_version('haproxy', '__init__.py'),
     packages=find_packages(),
-    install_requires=install_requires,
+    install_requires=requirements,
     tests_require=test_requirements,
     entry_points={
         'console_scripts':

tests/test_legacy_links.sh

@@ -76,7 +76,7 @@ rm_container web-a lb
 docker run -d --name web-a -e HOSTNAME="web-a" dockercloud/hello-world
 docker run -d --name lb --link web-a:web-a -e DEFAULT_SSL_CERT="$(awk 1 ORS='\\n' cert1.pem)" -p 443:443 haproxy
 wait_for_startup https://${DOCKER_HOST_IP}
-curl -sSfL --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 | grep -iF "SSL certificate problem" > /dev/null
+curl -sSfL --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1> /dev/null && exit $?
 curl -sSfL --cacert ca1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 | grep -iF 'My hostname is web-a' > /dev/null
 echo
 
@@ -86,9 +86,9 @@ docker run -d --name web-a -e HOSTNAME="web-a" dockercloud/hello-world
 docker run -d --name lb --link web-a:web-a -e DEFAULT_SSL_CERT="$(awk 1 ORS='\\n' cert1.pem)" -e CA_CERT="$(awk 1 ORS='\\n' ca0.pem)" -p 443:443 haproxy
 wait_for_startup https://${DOCKER_HOST_IP}
 echo "   Sending request without certificate"
-curl -sSfL --cacert ca1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null | grep 'handshake' > /dev/null
+curl -sSfL --cacert ca1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null && exit $?
 echo "   Sending request with a wrong certificate"
-curl -sSfL --cacert ca1.pem --cert cert1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null | grep 'alert unknown ca' > /dev/null
+curl -sSfL --cacert ca1.pem --cert cert1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null && exit $?
 echo "   Sending request with the correct certificate"
 curl -sSfL --cacert ca1.pem --cert cert0.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 | grep -iF 'My hostname is web-a' > /dev/null
 echo
@@ -98,7 +98,7 @@ rm_container web-a lb
 docker run -d --name web-a -e HOSTNAME="web-a" dockercloud/hello-world
 docker run -d --name lb --link web-a:web-a -e CERT_FOLDER="/certs/" -v $(pwd)/cert1.pem:/certs/cert1.pem -p 443:443 haproxy
 wait_for_startup https://${DOCKER_HOST_IP}
-curl -sSfL --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 | grep -iF "SSL certificate problem" > /dev/null
+curl -sSfL --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null && exit $?
 curl -sSfL --cacert ca1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 | grep -iF 'My hostname is web-a' > /dev/null
 echo
 
@@ -108,9 +108,9 @@ docker run -d --name web-a -e HOSTNAME="web-a" dockercloud/hello-world
 docker run -d --name lb --link web-a:web-a -e CERT_FOLDER="/certs/" -v $(pwd)/cert1.pem:/certs/cert1.pem -e CA_CERT_FILE="/cacert/ca0.pem"  -v $(pwd)/ca0.pem:/cacert/ca0.pem -p 443:443 haproxy
 wait_for_startup https://${DOCKER_HOST_IP}
 echo "   Sending request without certificate"
-curl -sSfL --cacert ca1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null | grep 'handshake' > /dev/null
+curl -sSfL --cacert ca1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null && exit $?
 echo "   Sending request with a wrong certificate"
-curl -sSfL --cacert ca1.pem --cert cert1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null | grep 'alert unknown ca' > /dev/null
+curl -sSfL --cacert ca1.pem --cert cert1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null && exit $?
 echo "   Sending request with the correct certificate"c
 curl -sSfL --cacert ca1.pem --cert cert0.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 | grep -iF 'My hostname is web-a' > /dev/null
 echo
@@ -132,9 +132,9 @@ docker run -d --name web-b -e HOSTNAME="web-b" -e VIRTUAL_HOST="https://web-b.or
 docker run -d --name lb  --link web-a:web-a --link web-b:web-b -p 443:443 haproxy
 wait_for_startup https://${DOCKER_HOST_IP}
 curl -sSfL --cacert ca1.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 | grep -iF 'My hostname is web-a' > /dev/null
-curl -sSfL --cacert ca2.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org  2>&1 2>&1 | grep -iF "SSL certificate problem: self signed certificate" > /dev/null
+curl -sSfL --cacert ca2.pem --resolve web-a.org:443:${DOCKER_HOST_IP} https://web-a.org 2>&1 > /dev/null && exit $?
 curl -sSfL --cacert ca2.pem --resolve web-b.org:443:${DOCKER_HOST_IP} https://web-b.org 2>&1 | grep -iF 'My hostname is web-b' > /dev/null
-curl -sSfL --cacert ca1.pem --resolve web-b.org:443:${DOCKER_HOST_IP} https://web-b.org 2>&1 2>&1 | grep -iF "SSL certificate problem: self signed certificate" > /dev/null
+curl -sSfL --cacert ca1.pem --resolve web-b.org:443:${DOCKER_HOST_IP} https://web-b.org 2>&1 > /dev/null && exit $?
 echo
 
 echo "=> Test multiple virtual host entries"