Sockets and gzip

Author: fikipollo

api/pysica_api_sockets.py

@@ -1,5 +1,6 @@
 import socket
 import json
+import zlib
 from struct import unpack, pack
 
 
@@ -28,6 +29,10 @@ def cache_add(element_id, data, data_type, timeout=None, compress=None, user_id=
     try:
         cp = SocketHandler()
         cp.connect(socket_file, server, port)
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to connect to cache server. Error message: " + str(ex)})
+
+    try:
         # Prepare package
         data = {
             'target': "add",
@@ -42,14 +47,18 @@ def cache_add(element_id, data, data_type, timeout=None, compress=None, user_id=
         response = cp.send_data(data, buffer_size=buffer_size)
         cp.close()
         return Response(json.loads(response))
-    except:
-        return Response({"success": False, "message": "Unable to connect to cache server."})
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to add element to cache. Error message: " + str(ex)})
 
 
 def cache_get(element_id=None, data_type=None, user_id=None, reset_timeout=False, timeout=None, socket_file=None, server="localhost", port=4444, buffer_size=4096):
     try:
         cp = SocketHandler()
         cp.connect(socket_file, server, port)
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to connect to cache server. Error message: " + str(ex)})
+
+    try:
         # Prepare package
         data = {
             'target': "get",
@@ -63,19 +72,22 @@ def cache_get(element_id=None, data_type=None, user_id=None, reset_timeout=False
         response = cp.get_data(data, buffer_size=buffer_size)
         cp.close()
         return Response(json.loads(response))
-    except:
-        return Response({"success": False, "message": "Unable to connect to cache server."})
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to get element from cache. Error message: " + str(ex)})
 
 
 def cache_remove(element_id, user_id=None, return_elem=False, socket_file=None, server="localhost", port=4444, buffer_size=4096):
+    # Get data
+    if return_elem:
+        element = cache_get(element_id=element_id, user_id=user_id, socket_file=socket_file, server=server, port=port, buffer_size=buffer_size)
+
     try:
-        # Get data
-        if return_elem:
-            element = cache_get(element_id=element_id, user_id=user_id, socket_file=socket_file, server=server, port=port, buffer_size=buffer_size)
-        # Remove element
         cp = SocketHandler()
         cp.connect(socket_file, server, port)
-        print("sadasds")
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to connect to cache server. Error message: " + str(ex)})
+
+    try:
         # Prepare package
         data={
             'target': "remove",
@@ -90,14 +102,18 @@ def cache_remove(element_id, user_id=None, return_elem=False, socket_file=None,
 
         cp.close()
         return Response(response)
-    except:
-        return Response({"success": False, "message": "Unable to connect to cache server."})
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to remove element from cache. Error message: " + str(ex)})
 
 
 def cache_reset(element_id, timeout=None, user_id=None, socket_file=None, server="localhost", port=4444, buffer_size=4096):
     try:
         cp = SocketHandler()
         cp.connect(socket_file, server, port)
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to connect to cache server. Error message: " + str(ex)})
+
+    try:
         # Prepare package
         data={
             'target': "reset",
@@ -109,16 +125,16 @@ def cache_reset(element_id, timeout=None, user_id=None, socket_file=None, server
         response = cp.send_data(data, buffer_size=buffer_size)
         cp.close()
         return Response(json.loads(response))
-    except:
-        return Response({"success": False, "message": "Unable to connect to cache server."})
+    except Exception as ex:
+        return Response({"success": False, "message": "Unable to reset element at cache. Error message: " + str(ex)})
 
 
 class SocketHandler:
     def __init__(self):
         self.socket = None
 
     def connect(self, socket_file=None, server_ip=None, server_port=None):
-        if socket_file is not None:
+        if socket_file is not None and socket_file != "":
             self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
             self.socket.connect(socket_file)
         else:
@@ -131,7 +147,7 @@ def close(self):
         self.socket = None
 
     def send_data(self, data, buffer_size=4096):
-        data = json.dumps(data)
+        data = zlib.compress(json.dumps(data))
         # use struct to make sure we have a consistent endianness on the length
         length = pack('>Q', len(data))
         # sendall to make sure it blocks if there's back-pressure on the socket
@@ -146,10 +162,10 @@ def send_data(self, data, buffer_size=4096):
             # to do it all in one go, so I believe.
             to_read = data_length - len(data)
             data += self.socket.recv(buffer_size if to_read > buffer_size else to_read)
-        return data
+        return zlib.decompress(data)
 
     def get_data(self, data, buffer_size=4096):
-        data = json.dumps(data)
+        data = zlib.compress(json.dumps(data))
         # use struct to make sure we have a consistent endianness on the length
         length = pack('>Q', len(data))
         # sendall to make sure it blocks if there's back-pressure on the socket
@@ -164,7 +180,7 @@ def get_data(self, data, buffer_size=4096):
             # to do it all in one go, so I believe.
             to_read = data_length - len(data)
             data += self.socket.recv(buffer_size if to_read > buffer_size else to_read)
-        return data
+        return zlib.decompress(data)
 
 
 class Response(object):
@@ -200,5 +216,3 @@ def __str__(self):
 
     def __repr__(self):
         return self.__str__()
-
-

docker/Dockerfile

@@ -12,8 +12,6 @@ MAINTAINER Rafael Hernandez <https://github.com/fikipollo>
 ################## BEGIN INSTALLATION ######################
 #INSTALL THE DEPENDENCIES
 
-COPY configs/* /tmp/
-
 RUN set -x \
     && apk --update --no-cache add make unzip \
     && apk --update --no-cache --virtual .build-dep add build-base linux-headers tzdata \
@@ -26,9 +24,12 @@ RUN set -x \
     && pip install --no-cache-dir APScheduler \
     && pip install --no-cache-dir https://github.com/unbit/uwsgi/archive/uwsgi-2.0.zip#egg=uwsgi \
     && apk del .build-dep \
-    && unzip /tmp/pysica.zip -d /tmp/pysica \
+    && mkdir -p /var/www/
+
+COPY configs/* /tmp/
+
+RUN unzip /tmp/pysica.zip -d /tmp/pysica \
     && mv /tmp/pysica_uwsgi.ini /var/ \
-    && mkdir -p /var/www/ \
     && mv /tmp/pysica /var/www/pysica \
     && mv /tmp/entrypoint.sh /usr/bin/entrypoint.sh \
     && chmod +x /usr/bin/entrypoint.sh \

docker/configs/entrypoint.sh

@@ -18,11 +18,10 @@ SERVER_HOST_NAME="${SERVER_HOST_NAME:-0.0.0.0}"
 
 # Change the default configuration based on environment
 sed -i 's/"SERVER_MODE".*/"SERVER_MODE" : "'${SERVER_MODE}'",/' /var/www/pysica/conf/server.cfg
-sed -i 's/"SERVER_SOCKET_FILE".*/"SERVER_SOCKET_FILE" : "'${SERVER_SOCKET_FILE}'",/' /var/www/pysica/conf/server.cfg
+sed -i 's#"SERVER_SOCKET_FILE".*#"SERVER_SOCKET_FILE" : "'${SERVER_SOCKET_FILE}'",#' /var/www/pysica/conf/server.cfg
 sed -i 's/"SERVER_BUFFER_SIZE".*/"SERVER_BUFFER_SIZE" : '${SERVER_BUFFER_SIZE}',/' /var/www/pysica/conf/server.cfg
 sed -i 's/"SERVER_HOST_NAME".*/"SERVER_HOST_NAME" : "'${SERVER_HOST_NAME}'",/' /var/www/pysica/conf/server.cfg
 
-
 echo "#---------------------------------------------------------------------------------------------------"
 echo "# Welcome to PySiCa (Python Simple Cache)"
 
@@ -33,7 +32,7 @@ if [[ "$SERVER_MODE" == "socket_based" ]]; then
     echo "# the server.cfg file. Available options are 'web_based' for RESTFUL mode based on HTTP "
     echo "# and 'socket_based' to use pure socket communication."
     echo "#---------------------------------------------------------------------------------------------------"
-    python /var/www/pysica/server_sockets.py
+    python /var/www/pysica/server_sockets.py &
 else
     echo "# Cache server is running as web server"
     echo "# To change the running mode, please set the value for the variable SERVER_MODE in the"
@@ -45,8 +44,8 @@ else
       kill -9 `cat /tmp/pysica.pid`
       rm /tmp/pysica.*
     fi
-
     uwsgi --ini /var/pysica_uwsgi.ini
-    tail -f /var/log/pysica/cache.log
 fi
 
+echo "Listening to /var/log/pysica/cache.log"
+tail -f /var/log/pysica/cache.log

docker/configs/pysica.zip

@@ -14,8 +14,8 @@ rm -rf venv docker test .git .idea
 rm conf/*
 rm *.pyc
 
-sed -i 's/\/tmp\/cache/\/var\/log\/pysica\/cache/g' default/server.default.cfg
-sed -i 's/"DEBUG" : true/"DEBUG" : false/g' default/server.default.cfg
+sed -i 's#/tmp/cache#/var/log/pysica/cache#g' default/server.default.cfg
+sed -i 's/"DEBUG".*/"DEBUG" : false,/' default/server.default.cfg
 
 echo "Compressing server application... "
 zip -r ../pysica.zip *

docker/deploy.sh

@@ -20,21 +20,12 @@ def __init__(self):
         # SERVER DEFINITION
         # ------------------------------------------------------------------------------------------
         self.settings = self.read_settings_file()
-
+        self.buffer_size = 0
         # Enable the logging to file for production
-        logger = None
-        if not self.settings.get("DEBUG", True):
-            try:
-                handler = RotatingFileHandler(self.settings.get("LOG_FILE"), maxBytes=31457280, backupCount=5)
-                handler.setLevel(logging.INFO)
-                handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s : %(funcName)s - %(message)s'))
-                logger = logging.getLogger(__name__)
-                logger.root.addHandler(handler)
-            except Exception as e:
-                raise Exception("Unable to open log file " + self.settings.get("LOG_FILE", "LOG FILE NOT SPECIFIED") + ". Error message: " + str(e))
-
+        self.logger= self.configure_logging()
+        # Create the instance for the cache
         self.cache_instance = PySiCa(
-            logger=logger,
+            logger=self.logger,
             timeout=self.settings.get("TIMEOUT"),
             compress=self.settings.get("COMPRESS"),
             max_elems=self.settings.get("MAX_ELEMS"),
@@ -136,6 +127,22 @@ def read_settings_file(self):
 
         return settings
 
+    def configure_logging(self):
+        if not self.settings.get("DEBUG", True):
+            try:
+                logger = logging.getLogger()
+                logger.handlers=[]
+                #
+                handler = RotatingFileHandler(self.settings.get("LOG_FILE"), maxBytes=31457280, backupCount=5)
+                handler.propagate = False
+                handler.setLevel(logging.INFO)
+                handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s : %(funcName)s - %(message)s'))
+                logger.root.addHandler(handler)
+                return logger
+            except Exception as e:
+                raise Exception("Unable to open log file " + self.settings.get("LOG_FILE", "LOG FILE NOT SPECIFIED") + ". Error message: " + str(e))
+        else:
+            return logging.root
 
 api = application = falcon.API()
 application.req_options.auto_parse_form_urlencoded = True