diff --git a/.github/actions/setup-ci/action.yaml b/.github/actions/setup-ci/action.yaml index e8a9425d8b..e9c0d335ec 100644 --- a/.github/actions/setup-ci/action.yaml +++ b/.github/actions/setup-ci/action.yaml @@ -46,9 +46,3 @@ runs: run: | sudo apt-get install -y libdigest-hmac-perl pip install 's3cmd==2.3.0' - - name: fix sproxyd.conf permissions - shell: bash - run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf - - name: ensure fuse kernel module is loaded (for sproxyd) - shell: bash - run: sudo modprobe fuse diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index e65cb73460..d73d870bb3 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -113,4 +113,6 @@ services: image: sproxyd-standalone build: ./sproxyd user: 0:0 - privileged: yes + privileged: yes # for setxattr on local filesystem + volumes: + - /tmp/artifacts/${JOB_NAME}/sproxyd:/logs diff --git a/.github/docker/sproxyd/Dockerfile b/.github/docker/sproxyd/Dockerfile index d337dbd97a..01daec09d0 100644 --- a/.github/docker/sproxyd/Dockerfile +++ b/.github/docker/sproxyd/Dockerfile @@ -1,3 +1,3 @@ -FROM ghcr.io/scality/federation/sproxyd:7.10.6.8 -ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/ -RUN chown root:root /conf/sproxyd0.conf +FROM ghcr.io/scality/federation/sproxyd:9.5.1.0 +ADD --chown=root:root ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/ +RUN mkdir -p /run0 diff --git a/.github/docker/sproxyd/conf/nginx.conf b/.github/docker/sproxyd/conf/nginx.conf index 6572f7caa0..9237eff615 100644 --- a/.github/docker/sproxyd/conf/nginx.conf +++ b/.github/docker/sproxyd/conf/nginx.conf @@ -3,7 +3,6 @@ error_log /logs/error.log; user root root; events { worker_connections 1000; - reuse_port on; multi_accept on; } worker_rlimit_nofile 20000; @@ -14,12 +13,25 @@ http { keepalive 40; server 127.0.0.1:20000; } + + # Define a custom log format + log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",' + '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,' + '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,' + '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",' + '"contentType":"$content_type","s3Address":"$remote_addr",' + '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",' + '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",' + '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",' + '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",' + '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,' + '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }'; + server { client_max_body_size 0; client_body_timeout 150; client_header_timeout 150; postpone_output 0; - client_body_postpone_size 0; keepalive_requests 1100; keepalive_timeout 300s; server_tokens off; @@ -28,7 +40,7 @@ http { tcp_nodelay on; tcp_nopush on; sendfile on; - listen 81; + listen 81 reuseport; server_name localhost; rewrite ^/arc/(.*)$ /dc1/$1 permanent; location ~* ^/proxy/(.*)$ { @@ -39,45 +51,36 @@ http { deny all; set $usermd '-'; set $sentusermd '-'; - set $elapsed_ms '-'; - set $now '-'; - log_by_lua ' - if not(ngx.var.http_x_scal_usermd == nil) and string.len(ngx.var.http_x_scal_usermd) > 2 then - ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3) - end + set $elapsed_ms '-'; + set $now '-'; + log_by_lua ' + if not(ngx.var.http_x_scal_usermd == nil) and string.len(ngx.var.http_x_scal_usermd) > 2 then + ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3) + end if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3) end - local elapsed_ms = tonumber(ngx.var.request_time) - if not ( elapsed_ms == nil) then - elapsed_ms = elapsed_ms * 1000 - ngx.var.elapsed_ms = tostring(elapsed_ms) - end - local time = tonumber(ngx.var.msec) * 1000 - ngx.var.now = time - '; - log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",' - '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,' - '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,' - '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",' - '"contentType":"$content_type","s3Address":"$remote_addr",' - '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",' - '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",' - '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",' - '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",' - '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,' - '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }'; + local elapsed_ms = tonumber(ngx.var.request_time) + if not ( elapsed_ms == nil) then + elapsed_ms = elapsed_ms * 1000 + ngx.var.elapsed_ms = tostring(elapsed_ms) + end + local time = tonumber(ngx.var.msec) * 1000 + ngx.var.now = time + '; access_log /dev/stdout irm; - error_log /dev/stdout error; + error_log /dev/stdout error; location / { + proxy_buffering off; proxy_request_buffering off; + # proxy_http_version 1.1; Option disabled as it affects performance cf citadel design fastcgi_request_buffering off; fastcgi_no_cache 1; fastcgi_cache_bypass 1; fastcgi_buffering off; fastcgi_ignore_client_abort on; fastcgi_keep_conn on; - include fastcgi_params; + include /conf/fastcgi_params; fastcgi_pass sproxyds; fastcgi_next_upstream error timeout; fastcgi_send_timeout 285s; diff --git a/.github/docker/sproxyd/conf/supervisord.conf b/.github/docker/sproxyd/conf/supervisord.conf index 13608f437b..121f8e4e5f 100644 --- a/.github/docker/sproxyd/conf/supervisord.conf +++ b/.github/docker/sproxyd/conf/supervisord.conf @@ -17,7 +17,7 @@ serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock [program:nginx] directory=%(ENV_SUP_RUN_DIR)s -command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'" +command=bash -c "/usr/bin/openresty -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'" stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log stdout_logfile_maxbytes=100MB @@ -27,13 +27,22 @@ stderr_logfile_backups=7 autorestart=true autostart=true user=root +priority=40 [program:sproxyd] directory=%(ENV_SUP_RUN_DIR)s process_name=%(program_name)s-%(process_num)s numprocs=1 numprocs_start=0 -command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s +# -d no daemon +# -l log on console +# -w no watchdog +# -V127 log level info +# -c config file +# -P sysfs confpath +# -A adminapi directory +# -s no sysfs do not mount FUSE sysfs +command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s -A /run%(process_num)s -s stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log stdout_logfile_maxbytes=100MB stdout_logfile_backups=7 @@ -41,3 +50,4 @@ redirect_stderr=true autorestart=true autostart=true user=root +priority=20