Fuseki rate limiting

Dockerfile

@@ -111,7 +111,7 @@ ENV MAX_REQUEST_RETRIES=3
 
 ENV IMPORT_KEEPALIVE=
 
-ENV MAX_IMPORT_THREADS=10
+ENV MAX_IMPORT_THREADS=1
 
 ENV SERVLET_NAME=
 

docker-compose.yml

@@ -34,7 +34,7 @@ services:
       - SSL_VERIFY_CLIENT=optional_no_ca
       - MAX_BODY_SIZE=2097152
     volumes:
-      - ./platform/nginx.conf.template:/etc/nginx/nginx.conf.template:ro
+      - ./platform/nginx-frontend.conf.template:/etc/nginx/nginx.conf.template:ro
       - ./ssl/server:/etc/nginx/ssl:ro
   linkeddatahub:
     user: root # otherwise the ldh user does not have permissions to the mounted folder which is owner by root
@@ -63,6 +63,7 @@ services:
       - SELF_SIGNED_CERT=true # only on localhost
       - SIGN_UP_CERT_VALIDITY=180
       - MAX_CONTENT_LENGTH=2097152
+      - MAX_IMPORT_THREADS=1
       - NOTIFICATION_ADDRESS=LinkedDataHub <notifications@localhost>
       - MAIL_SMTP_HOST=email-server
       - MAIL_SMTP_PORT=25
@@ -133,10 +134,11 @@ services:
     user: root # otherwise the varnish user does not have permissions to the mounted folder which is owner by root
     depends_on:
       - linkeddatahub
+      - nginx-admin
     tmpfs: /var/lib/varnish/varnishd:exec
     environment:
-      - BACKEND_HOST=fuseki-admin
-      - BACKEND_PORT=3030
+      - BACKEND_HOST=nginx-admin
+      - BACKEND_PORT=8080
       - CLIENT_HOST=linkeddatahub
       - VARNISH_SIZE=1G
     entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint \"$$0\" \"$$@\""
@@ -148,16 +150,39 @@ services:
     user: root # otherwise varnish user does not have permissions to the mounted folder which is owner by root
     depends_on:
       - linkeddatahub
+      - nginx-end-user
     tmpfs: /var/lib/varnish/varnishd:exec
     environment:
-      - BACKEND_HOST=fuseki-end-user
-      - BACKEND_PORT=3030
+      - BACKEND_HOST=nginx-end-user
+      - BACKEND_PORT=8080
       - CLIENT_HOST=linkeddatahub
       - VARNISH_SIZE=1G
     entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint \"$$0\" \"$$@\""
     command: [ "-t", "86400", "-p", "timeout_idle=60s" ] # time to live
     volumes:
       - ./platform/varnish-backend.vcl.template:/etc/varnish/default.vcl.template:ro
+  nginx-admin:
+    image: nginx:1.23.3
+    depends_on:
+      - fuseki-admin
+    command: /bin/sh -c "cp /etc/nginx/nginx.conf.template /etc/nginx/nginx.conf && sed -i 's|$${UPSTREAM_SERVER}|'"$$UPSTREAM_SERVER"'|g' /etc/nginx/nginx.conf && sed -i 's|$${UPSTREAM_HTTP_PORT}|'"$$UPSTREAM_HTTP_PORT"'|g' /etc/nginx/nginx.conf && sed -i 's|$${SERVER_HTTP_PORT}|'"$$SERVER_HTTP_PORT"'|g' /etc/nginx/nginx.conf && nginx -g 'daemon off;'"
+    environment:
+      - UPSTREAM_SERVER=fuseki-admin
+      - UPSTREAM_HTTP_PORT=3030
+      - SERVER_HTTP_PORT=8080 # because of nginx-unprivileged
+    volumes:
+      - ./platform/nginx-backend.conf.template:/etc/nginx/nginx.conf.template:ro
+  nginx-end-user:
+    image: nginx:1.23.3
+    depends_on:
+      - fuseki-end-user
+    command: /bin/sh -c "cp /etc/nginx/nginx.conf.template /etc/nginx/nginx.conf && sed -i 's|$${UPSTREAM_SERVER}|'"$$UPSTREAM_SERVER"'|g' /etc/nginx/nginx.conf && sed -i 's|$${UPSTREAM_HTTP_PORT}|'"$$UPSTREAM_HTTP_PORT"'|g' /etc/nginx/nginx.conf && sed -i 's|$${SERVER_HTTP_PORT}|'"$$SERVER_HTTP_PORT"'|g' /etc/nginx/nginx.conf && nginx -g 'daemon off;'"
+    environment:
+      - UPSTREAM_SERVER=fuseki-end-user
+      - UPSTREAM_HTTP_PORT=3030
+      - SERVER_HTTP_PORT=8080 # because of nginx-unprivileged
+    volumes:
+      - ./platform/nginx-backend.conf.template:/etc/nginx/nginx.conf.template:ro
   email-server:
     image: namshi/smtp
     environment:

platform/nginx-backend.conf.template

@@ -0,0 +1,40 @@
+worker_processes 1;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    # Rate limiting: 5 requests per second per IP
+    limit_req_zone $binary_remote_addr zone=api_ratelimit:10m rate=5r/s;
+    limit_req_status 429;
+
+    upstream varnish_backend {
+        server ${UPSTREAM_SERVER}:${UPSTREAM_HTTP_PORT};
+    }
+
+    server {
+        listen ${SERVER_HTTP_PORT};
+
+        # Health check path
+        location = /healthz {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+
+        location / {
+            # Apply rate limiting with burst buffer
+            limit_req zone=api_ratelimit burst=10;
+            # Optional: tell clients how long to wait (1s = 5r/s baseline)
+            add_header Retry-After 1 always;
+
+            proxy_pass http://varnish_backend;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+}

platform/varnish-backend.vcl.template

@@ -54,6 +54,11 @@ sub vcl_recv {
 }
 
 sub vcl_backend_response {
+    if (beresp.status == 429) {
+        set beresp.uncacheable = true;
+        return (deliver);
+    }
+
     /* purge URLs after updates */
     if ((beresp.status == 200 || beresp.status == 201 || beresp.status == 204) && bereq.method ~ "POST|PUT|DELETE|PATCH") {
         set beresp.http.X-LinkedDataHub = "Banned";

src/main/java/com/atomgraph/linkeddatahub/client/GraphStoreClient.java

@@ -0,0 +1,197 @@
+/**
+ *  Copyright 2025 Martynas Jusevičius <martynas@atomgraph.com>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package com.atomgraph.linkeddatahub.client;
+
+import com.atomgraph.core.MediaTypes;
+import com.atomgraph.linkeddatahub.client.util.RetryingInvocationBuilder;
+import jakarta.ws.rs.NotFoundException;
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.client.WebTarget;
+import jakarta.ws.rs.core.MultivaluedHashMap;
+import jakarta.ws.rs.core.MultivaluedMap;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.Response.Status;
+import org.apache.jena.rdf.model.Model;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author Martynas.Jusevicius
+ */
+public class GraphStoreClient extends com.atomgraph.core.client.GraphStoreClient {
+
+    private static final Logger log = LoggerFactory.getLogger(GraphStoreClient.class);
+
+    private final long defaultDelayMillis; // = 5000L;
+    private final int maxRetryCount; // = 3;
+
+    protected GraphStoreClient(MediaTypes mediaTypes, WebTarget endpoint, long defaultDelayMillis, int maxRetryCount) {
+        super(mediaTypes, endpoint);
+        this.defaultDelayMillis = defaultDelayMillis;
+        this.maxRetryCount = maxRetryCount;
+    }
+
+    public static GraphStoreClient create(MediaTypes mediaTypes, WebTarget endpoint, long defaultDelayMillis, int maxRetryCount) {
+        return new GraphStoreClient(mediaTypes, endpoint, defaultDelayMillis, maxRetryCount);
+    }
+
+    @Override
+    public boolean containsModel(String uri) {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(GRAPH_PARAM_NAME, uri);
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(getReadableMediaTypes(Model.class)),
+                defaultDelayMillis,
+                maxRetryCount
+        ).head()) {
+            return cr.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL);
+        }
+    }
+
+    @Override
+    public Model getModel() {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(DEFAULT_PARAM_NAME, Boolean.TRUE.toString());
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(getReadableMediaTypes(Model.class)),
+                defaultDelayMillis,
+                maxRetryCount
+        ).get()) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+            return cr.readEntity(Model.class);
+        }
+    }
+
+    @Override
+    public Model getModel(String uri) {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(GRAPH_PARAM_NAME, uri);
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(getReadableMediaTypes(Model.class)),
+                defaultDelayMillis,
+                maxRetryCount
+        ).get()) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+            return cr.readEntity(Model.class);
+        }
+    }
+
+    @Override
+    public void add(Model model) {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(DEFAULT_PARAM_NAME, Boolean.TRUE.toString());
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(),
+                defaultDelayMillis,
+                maxRetryCount
+        ).post(Entity.entity(model, getDefaultMediaType()))) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+        }
+    }
+
+    @Override
+    public void add(String uri, Model model) {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(GRAPH_PARAM_NAME, uri);
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(),
+                defaultDelayMillis,
+                maxRetryCount
+        ).post(Entity.entity(model, getDefaultMediaType()))) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+        }
+    }
+
+    @Override
+    public void putModel(Model model) {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(DEFAULT_PARAM_NAME, Boolean.TRUE.toString());
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(),
+                defaultDelayMillis,
+                maxRetryCount
+        ).put(Entity.entity(model, getDefaultMediaType()))) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+        }
+    }
+
+    @Override
+    public void putModel(String uri, Model model) {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(GRAPH_PARAM_NAME, uri);
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(),
+                defaultDelayMillis,
+                maxRetryCount
+        ).put(Entity.entity(model, getDefaultMediaType()))) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+        }
+    }
+
+    @Override
+    public void deleteDefault() {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(DEFAULT_PARAM_NAME, Boolean.TRUE.toString());
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(),
+                defaultDelayMillis,
+                maxRetryCount
+        ).delete()) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+        }
+    }
+
+    @Override
+    public void deleteModel(String uri) {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle(GRAPH_PARAM_NAME, uri);
+
+        try (Response cr = new RetryingInvocationBuilder(
+                applyParams(params).request(),
+                defaultDelayMillis,
+                maxRetryCount
+        ).delete()) {
+            if (cr.getStatus() == Status.NOT_FOUND.getStatusCode()) {
+                throw new NotFoundException();
+            }
+        }
+    }
+
+}