Implement HTTP gzip/zstd Compression and WebSocket permessage-deflate Support

Author: ProjectSky

.github/workflows/CI.yml

@@ -19,8 +19,8 @@ jobs:
       - name: build
         run: |
           sudo apt update
-          sudo apt install libssl-dev libnghttp2-dev
-          ./configure --with-openssl --with-nghttp2 --with-kcp --with-mqtt
+          sudo apt install libssl-dev libnghttp2-dev zlib1g-dev libzstd-dev
+          ./configure --with-openssl --with-nghttp2 --with-kcp --with-mqtt --with-zlib --with-zstd
           make libhv evpp
 
       - name: test

BUILD.bazel

@@ -1,4 +1,4 @@
-load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 
 
 config_setting(
@@ -85,6 +85,16 @@ config_setting(
     visibility = [":__subpackages__"]
 )
 
+config_setting(
+    name = "with_http_server_client",
+    define_values = {
+        "WITH_EVPP": "ON",
+        "WITH_HTTP": "ON",
+        "WITH_HTTP_SERVER": "ON",
+        "WITH_HTTP_CLIENT": "ON",
+    },
+)
+
 config_setting(
     name = "with_evpp_nghttp2",
     define_values = {
@@ -120,6 +130,16 @@ config_setting(
     define_values = {"WITH_NGHTTP2": "ON"}
 )
 
+config_setting(
+    name = "with_zlib",
+    define_values = {"WITH_ZLIB": "ON"}
+)
+
+config_setting(
+    name = "with_zstd",
+    define_values = {"WITH_ZSTD": "ON"}
+)
+
 config_setting(
     name = "with_openssl",
     define_values = {"WITH_OPENSSL": "ON"}
@@ -219,6 +239,12 @@ COPTS = select({
 }) + select({
     "with_nghttp2": ["-DWITH_NGHTTP2"],
     "//conditions:default": [],
+}) + select({
+    "with_zlib": ["-DWITH_ZLIB"],
+    "//conditions:default": [],
+}) + select({
+    "with_zstd": ["-DWITH_ZSTD"],
+    "//conditions:default": [],
 }) + select({
     "with_openssl": ["-DWITH_OPENSSL"],
     "//conditions:default": [],
@@ -253,6 +279,12 @@ LINKOPTS = select({
 }) + select({
     "@bazel_tools//tools/cpp:gcc": ["-lrt"],
     "//conditions:default": [],
+}) + select({
+    "with_zlib": ["-lz"],
+    "//conditions:default": [],
+}) + select({
+    "with_zstd": ["-lzstd"],
+    "//conditions:default": [],
 })
 
 BASE_HEADERS = [
@@ -332,6 +364,7 @@ PROTOCOL_HEADERS = [
 HTTP_HEADERS = [
     "http/httpdef.h",
     "http/wsdef.h",
+    "http/HttpCompression.h",
     "http/http_content.h",
     "http/HttpMessage.h",
     "http/HttpParser.h",
@@ -449,6 +482,32 @@ cc_library(
     visibility = ["//visibility:public"]
 )
 
+cc_test(
+    name = "http_compression_test",
+    srcs = ["unittest/http_compression_test.cpp"],
+    copts = COPTS,
+    deps = [":hv"],
+)
+
+cc_test(
+    name = "websocket_compression_test",
+    srcs = ["unittest/websocket_compression_test.cpp"],
+    copts = COPTS,
+    deps = [":hv"],
+)
+
+filegroup(
+    name = "unittests",
+    srcs = select({
+        "with_http_server_client": [
+            ":http_compression_test",
+            ":websocket_compression_test",
+        ],
+        "//conditions:default": [],
+    }),
+    visibility = ["//:__pkg__"],
+)
+
 filegroup(
     name = "libhv",
     srcs = select({
@@ -460,7 +519,8 @@ filegroup(
     })  + select({
         "build_examples": ["//examples:examples"],
         "//conditions:default": [],
+    }) + select({
+        "build_unittest": [":unittests"],
+        "//conditions:default": [],
     })
 )
-
-

BUILD.md

@@ -144,6 +144,20 @@ bin/httpd -s restart -d
 bin/curl -v http://localhost:8080 --http2
 ```
 
+### compile WITH_ZLIB
+```
+sudo apt install zlib1g-dev # ubuntu
+./configure --with-zlib
+make clean && make
+```
+
+### compile WITH_ZSTD (Requires libzstd >= 1.5.0)
+```
+sudo apt install libzstd-dev # ubuntu
+./configure --with-zstd
+make clean && make
+```
+
 ### compile WITH_KCP
 ```
 ./configure --with-kcp

CMakeLists.txt

@@ -23,6 +23,8 @@ option(USE_MULTIMAP "MultiMap" OFF)
 
 option(WITH_CURL "with curl library (deprecated)" OFF)
 option(WITH_NGHTTP2 "with nghttp2 library" OFF)
+option(WITH_ZLIB "with zlib library" OFF)
+option(WITH_ZSTD "with zstd library" OFF)
 
 option(WITH_OPENSSL "with openssl library" OFF)
 option(WITH_GNUTLS  "with gnutls library"  OFF)
@@ -152,6 +154,16 @@ if(WITH_NGHTTP2)
     set(LIBS ${LIBS} nghttp2)
 endif()
 
+if(WITH_ZLIB)
+    add_definitions(-DWITH_ZLIB)
+    set(LIBS ${LIBS} z)
+endif()
+
+if(WITH_ZSTD)
+    add_definitions(-DWITH_ZSTD)
+    set(LIBS ${LIBS} zstd)
+endif()
+
 if(WITH_OPENSSL)
     add_definitions(-DWITH_OPENSSL)
     find_package(OpenSSL)

Makefile

@@ -286,6 +286,12 @@ unittest: prepare
 	$(CC)  -g -Wall -O0 -std=c99   -I. -Ibase -Iprotocol -o bin/ping              unittest/ping_test.c          protocol/icmp.c base/hsocket.c base/htime.c -DPRINT_DEBUG
 	$(CC)  -g -Wall -O0 -std=c99   -I. -Ibase -Iprotocol -o bin/ftp               unittest/ftp_test.c           protocol/ftp.c  base/hsocket.c base/htime.c
 	$(CC)  -g -Wall -O0 -std=c99   -I. -Ibase -Iprotocol -Iutil -o bin/sendmail   unittest/sendmail_test.c      protocol/smtp.c base/hsocket.c base/htime.c util/base64.c
+	@if [ "$(WITH_HTTP)" = "yes" ] && [ "$(WITH_HTTP_CLIENT)" = "yes" ] && [ "$(WITH_HTTP_SERVER)" = "yes" ]; then \
+		$(MAKEF) TARGET=http_compression_test SRCDIRS="$(CORE_SRCDIRS) util cpputil evpp http http/client http/server" \
+			SRCS="unittest/http_compression_test.cpp"; \
+		$(MAKEF) TARGET=websocket_compression_test SRCDIRS="$(CORE_SRCDIRS) util cpputil evpp http http/client http/server" \
+			SRCS="unittest/websocket_compression_test.cpp"; \
+	fi
 
 run-unittest: unittest
 	bash scripts/unittest.sh

Makefile.in

@@ -182,6 +182,16 @@ ifeq ($(WITH_NGHTTP2), yes)
 	LDFLAGS += -lnghttp2
 endif
 
+ifeq ($(WITH_ZLIB), yes)
+	CPPFLAGS += -DWITH_ZLIB
+	LDFLAGS += -lz
+endif
+
+ifeq ($(WITH_ZSTD), yes)
+	CPPFLAGS += -DWITH_ZSTD
+	LDFLAGS += -lzstd
+endif
+
 ifeq ($(WITH_OPENSSL), yes)
 	CPPFLAGS += -DWITH_OPENSSL
 	LDFLAGS += -lssl -lcrypto

Makefile.vars

@@ -71,6 +71,7 @@ PROTOCOL_HEADERS =  protocol/icmp.h\
 
 HTTP_HEADERS =  http/httpdef.h\
 				http/wsdef.h\
+				http/HttpCompression.h\
 				http/http_content.h\
 				http/HttpMessage.h\
 				http/HttpParser.h\

base/herr.h

@@ -35,11 +35,14 @@
     F(1017, INVALID_FMT,        "Invalid format")   \
     F(1018, INVALID_PROTOCOL,   "Invalid protocol") \
     F(1019, INVALID_PACKAGE,    "Invalid package")  \
+    F(1020, UNSUPPORTED_CONTENT_ENCODING, "Unsupported content encoding") \
     \
     F(1021, OUT_OF_RANGE,   "Out of range")     \
     F(1022, OVER_LIMIT,     "Over the limit")       \
     F(1023, MISMATCH,       "Mismatch")         \
     F(1024, PARSE,          "Parse failed")     \
+    F(1025, COMPRESS,       "Compress failed")  \
+    F(1026, DECOMPRESS,     "Decompress failed")\
     \
     F(1030, OPEN_FILE,      "Open file failed") \
     F(1031, SAVE_FILE,      "Save file failed") \

cmake/vars.cmake

@@ -78,6 +78,7 @@ set(PROTOCOL_HEADERS
 set(HTTP_HEADERS
     http/httpdef.h
     http/wsdef.h
+    http/HttpCompression.h
     http/http_content.h
     http/HttpMessage.h
     http/HttpParser.h

config.ini

@@ -31,6 +31,10 @@ USE_MULTIMAP=no
 WITH_CURL=no
 # for http2
 WITH_NGHTTP2=no
+# for http gzip / websocket permessage-deflate
+WITH_ZLIB=no
+# for http zstd
+WITH_ZSTD=no
 # for SSL/TLS
 WITH_OPENSSL=no
 WITH_GNUTLS=no