Version 5.x

.github/workflows/http-tests.yml

@@ -7,37 +7,49 @@ jobs:
     name: Build Docker image and run HTTP test suite against it
     runs-on: ubuntu-latest
     env:
-      ASF_ARCHIVE: http://archive.apache.org/dist/
+      ASF_ARCHIVE: https://archive.apache.org/dist/
       JENA_VERSION: 4.7.0
       BASE_URI: https://localhost:4443/
-      OWNER_CERT_PWD: changeit
-      SECRETARY_CERT_PWD: LinkedDataHub
     steps:
       - name: Install Linux packages
         run: sudo apt-get update && sudo apt-get install -qq raptor2-utils && sudo apt-get install curl
       - name: Download Jena
-        run: curl -sS --fail "${{env.ASF_ARCHIVE}}jena/binaries/apache-jena-${{env.JENA_VERSION}}.tar.gz" -o "${{runner.temp}}/jena.tar.gz"
+        run: curl -sS --fail "${{ env.ASF_ARCHIVE }}jena/binaries/apache-jena-${{ env.JENA_VERSION }}.tar.gz" -o "${{ runner.temp }}/jena.tar.gz"
       - name: Unpack Jena
         run: tar -zxf jena.tar.gz
-        working-directory: ${{runner.temp}}
-      - run: echo "$JENA_HOME/bin" >> $GITHUB_PATH
-        env:
-            JENA_HOME: "${{runner.temp}}/apache-jena-${{env.JENA_VERSION}}"
+        working-directory: ${{ runner.temp }}
+      - name: Set JENA_HOME and update PATH
+        run: |
+          echo "${{ runner.temp }}/apache-jena-${{ env.JENA_VERSION }}/bin" >> $GITHUB_PATH
       - name: Checkout code
         uses: actions/checkout@v3
-      - name: Generate owner's and secretary's certificates/public keys
-        run: ../scripts/setup.sh .env ssl "${{env.OWNER_CERT_PWD}}" "${{env.SECRETARY_CERT_PWD}}" 3650
-        shell: bash
+      - name: Add bin/ and its subdirectories to PATH
+        run: |
+          find "$GITHUB_WORKSPACE/bin" -type d >> "$GITHUB_PATH"
+      - name: Generating server certificate
+        run: |
+          server-cert-gen.sh .env nginx ssl
         working-directory: http-tests
+      - name: Writing secrets to files
+        run: |
+          mkdir -p ./secrets
+          printf "%s" "${{ secrets.HTTP_TEST_OWNER_CERT_PASSWORD }}" > ./secrets/owner_cert_password.txt
+          printf "%s" "${{ secrets.HTTP_TEST_SECRETARY_CERT_PASSWORD }}" > ./secrets/secretary_cert_password.txt
+          printf "%s" "${{ secrets.HTTP_TEST_SECRETARY_CERT_PASSWORD }}" > ./secrets/client_truststore_password.txt
+        shell: bash
       - name: Build Docker image & Run Docker containers
-        run: docker-compose -f docker-compose.yml -f ./http-tests/docker-compose.http-tests.yml --env-file ./http-tests/.env up --build -d
+        run: docker compose -f docker-compose.yml -f ./http-tests/docker-compose.http-tests.yml --env-file ./http-tests/.env up --build -d
       - name: Wait for the server to start...
         run: while ! (status=$(curl -k -s -w "%{http_code}\n" https://localhost:4443 -o /dev/null) && echo "$status" && echo "$status" | grep "403") ; do sleep 1 ; done # wait for the webapp to start (returns 403 by default)
+      - name: Fix certificate permissions on the host
+        run: |
+          sudo chmod 644 ./ssl/owner/cert.pem ./ssl/secretary/cert.pem
+        working-directory: http-tests
       - name: Run HTTP test scripts
-        run: ./run.sh "$PWD/ssl/owner/cert.pem" "${{env.OWNER_CERT_PWD}}" "$PWD/ssl/secretary/cert.pem" "${{env.SECRETARY_CERT_PWD}}"
+        run: ./run.sh "$PWD/ssl/owner/cert.pem" "${{ secrets.HTTP_TEST_OWNER_CERT_PASSWORD }}" "$PWD/ssl/secretary/cert.pem" "${{ secrets.HTTP_TEST_SECRETARY_CERT_PASSWORD }}"
         shell: bash
         working-directory: http-tests
       - name: Stop Docker containers
-        run: docker-compose --env-file ./http-tests/.env down
+        run: docker compose --env-file ./http-tests/.env down
       - name: Remove Docker containers
-        run: docker-compose --env-file ./http-tests/.env rm -f
+        run: docker compose --env-file ./http-tests/.env rm -f

.github/workflows/image.yml

@@ -0,0 +1,44 @@
+name: ci
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract version parts
+        id: version
+        run: |
+          VERSION="${{ github.ref_name }}"
+          MAJOR="${VERSION%%.*}"
+          MINOR="${VERSION%.*}"
+          MINOR="${MINOR#*.}"
+          echo "MAJOR=$MAJOR" >> $GITHUB_ENV
+          echo "MINOR=$MAJOR.$MINOR" >> $GITHUB_ENV
+          echo "FULL_VERSION=$VERSION" >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            atomgraph/linkeddatahub:latest
+            atomgraph/linkeddatahub:${{ env.FULL_VERSION }}
+            atomgraph/linkeddatahub:${{ env.MINOR }}
+            atomgraph/linkeddatahub:${{ env.MAJOR }}

.gitignore

@@ -1,10 +1,15 @@
+.DS_Store
 /target
-/certs
 /.env
 /data
 /uploads
 /nb-configuration.xml
 /node
 /node_modules
 /docker-compose.override.yml
+/secrets
+/datasets
 /ssl
+/http-tests/ssl
+/http-tests/datasets
+/http-tests/uploads

Dockerfile

@@ -1,4 +1,4 @@
-FROM maven:3.8.4-openjdk-17 as maven
+FROM maven:3.8.4-openjdk-17 AS maven
 
 # download and extract Jena
 
@@ -22,7 +22,7 @@ RUN mvn -Pstandalone clean install
 
 # ==============================
 
-FROM atomgraph/letsencrypt-tomcat:10.1.4
+FROM atomgraph/letsencrypt-tomcat:10.1.34
 
 LABEL maintainer="martynas@atomgraph.com"
 
@@ -38,10 +38,6 @@ ENV SOURCE_COMMIT=$SOURCE_COMMIT
 
 WORKDIR $CATALINA_HOME
 
-# add XSLT stylesheet that makes changes to ROOT.xml
-
-COPY platform/context.xsl conf/context.xsl
-
 ENV CACHE_MODEL_LOADS=true
 
 ENV STYLESHEET=static/com/atomgraph/linkeddatahub/xsl/bootstrap/2.3.2/layout.xsl
@@ -72,17 +68,26 @@ ENV HTTPS=false
 
 ENV SERVER_CERT=/var/linkeddatahub/ssl/server/server.crt
 
-ENV SECRETARY_CERT=/var/linkeddatahub/ssl/secretary/cert.pem
+ENV OWNER_CERT_ALIAS=root-owner
+ENV OWNER_KEYSTORE=/var/linkeddatahub/ssl/owner/keystore.p12
+ENV OWNER_CERT=/var/linkeddatahub/ssl/owner/cert.pem
+ENV OWNER_PUBLIC_KEY=/var/linkeddatahub/ssl/owner/public.pem
+ENV OWNER_PRIVATE_KEY=/var/linkeddatahub/ssl/owner/private.key
 
-ENV SECRETARY_CERT_ALIAS=secretary
+ENV SECRETARY_COMMON_NAME=LinkedDataHub
+ENV SECRETARY_CERT_ALIAS=root-secretary
+ENV SECRETARY_KEYSTORE=/var/linkeddatahub/ssl/secretary/keystore.p12
+ENV SECRETARY_CERT=/var/linkeddatahub/ssl/secretary/cert.pem
+ENV SECRETARY_PUBLIC_KEY=/var/linkeddatahub/ssl/secretary/public.pem
+ENV SECRETARY_PRIVATE_KEY=/var/linkeddatahub/ssl/secretary/private.key
 
 ENV CLIENT_KEYSTORE_MOUNT=/var/linkeddatahub/ssl/secretary/keystore.p12
-
 ENV CLIENT_KEYSTORE="$CATALINA_HOME/webapps/ROOT/WEB-INF/keystore.p12"
-
 ENV CLIENT_TRUSTSTORE="$CATALINA_HOME/webapps/ROOT/WEB-INF/client.truststore"
 
-ENV OWNER_PUBLIC_KEY=/var/linkeddatahub/ssl/owner/public.pem
+ENV CERT_VALIDITY=3650
+
+ENV SIGN_UP_CERT_VALIDITY=
 
 ENV LOAD_DATASETS=
 
@@ -102,11 +107,13 @@ ENV MAX_CONN_PER_ROUTE=20
 
 ENV MAX_TOTAL_CONN=40
 
+ENV MAX_REQUEST_RETRIES=3
+
 ENV IMPORT_KEEPALIVE=
 
-ENV GOOGLE_CLIENT_ID=
+ENV MAX_IMPORT_THREADS=10
 
-ENV GOOGLE_CLIENT_SECRET=
+ENV SERVLET_NAME=
 
 ENV GENERATE_SITEMAP=true
 
@@ -120,6 +127,14 @@ RUN apt-get update --allow-releaseinfo-change && \
     rm -rf webapps/* && \
     rm -rf /var/lib/apt/lists/*
 
+# add XSLT stylesheet that makes changes to ROOT.xml
+
+COPY platform/context.xsl /var/linkeddatahub/xsl/context.xsl
+
+# add XSLT stylesheet that makes changes to web.xml
+
+COPY platform/web.xsl /var/linkeddatahub/xsl/web.xsl
+
 # copy entrypoint
 
 COPY platform/entrypoint.sh entrypoint.sh
@@ -172,6 +187,7 @@ ENV PATH="${PATH}:${JENA_HOME}/bin"
 
 RUN useradd --no-log-init -U ldh && \
     chown -R ldh:ldh . && \
+    mkdir -p "$(dirname "$OIDC_REFRESH_TOKENS")" && \
     chown -R ldh:ldh /var/linkeddatahub && \
     mkdir -p "${UPLOAD_ROOT}/${UPLOAD_CONTAINER_PATH}" && \
     chown -R ldh:ldh "$UPLOAD_ROOT" && \
@@ -180,7 +196,7 @@ RUN useradd --no-log-init -U ldh && \
 
 RUN ./import-letsencrypt-stg-roots.sh
 
-HEALTHCHECK --start-period=80s --interval=20s --timeout=10s \
+HEALTHCHECK --start-period=80s --retries=5 \
     CMD curl -f -I "http://localhost:${HTTP_PORT}/ns" -H "Accept: application/n-triples" || exit 1 # relies on public access to the namespace document
 
 USER ldh

README.md

@@ -57,19 +57,23 @@ It takes a few clicks and filling out a form to install the product into your ow
      OWNER_STATE_OR_PROVINCE=Denmark
      OWNER_COUNTRY_NAME=DK
      ```
-  3. Setup SSL certificates/keys by running this from command line (replace `$owner_cert_pwd` and `$secretary_cert_pwd` with your own passwords):
-     ```
-     ./scripts/setup.sh .env ssl $owner_cert_pwd $secretary_cert_pwd 3650
+  3. Setup server's SSL certificates by running this from command line:
+     ```shell
+     ./scripts/server-cert-gen.sh .env nginx ssl
      ```
      The script will create an `ssl` sub-folder where the SSL certificates and/or public keys will be placed.
   4. Launch the application services by running this from command line:
-     ```
+     ```shell
      docker-compose up --build
      ```
      It will build LinkedDataHub's Docker image, start its container and mount the following sub-folders:
+     - `ssl`
+       * `owner` stores root owner's WebID certificate, keystore, and public key
+       * `secretary` stores root application's WebID certificate, keystore, and public key
+       * `server` stores the server's certificate (also used by nginx)
      - `data` where the triplestore(s) will persist RDF data
      - `uploads` where LDH stores content-hashed file uploads
-     The first should take around half a minute as datasets are being loaded into triplestores. After a successful startup, the last line of the Docker log should read something like:
+     It should take up to half a minute as datasets are being loaded into triplestores. After a successful startup, the last line of the Docker log should read something like:
      ```
      linkeddatahub_1     | 09-Feb-2021 14:18:10.536 INFO [main] org.apache.catalina.startup.Catalina.start Server startup in [32609] milliseconds
      ```
@@ -78,7 +82,15 @@ It takes a few clicks and filling out a form to install the product into your ow
      - Mozilla Firefox: `Options > Privacy > Security > View Certificates... > Import...`
      - Apple Safari: The file is installed directly into the operating system. Open the file and import it using the [Keychain Access](https://support.apple.com/guide/keychain-access/what-is-keychain-access-kyca1083/mac) tool (drag it to the `local` section).
      - Microsoft Edge: Does not support certificate management, you need to install the file into Windows. [Read more here](https://social.technet.microsoft.com/Forums/en-US/18301fff-0467-4e41-8dee-4e44823ed5bf/microsoft-edge-browser-and-ssl-certificates?forum=win10itprogeneral).
-  6. Open **https://localhost:4443/** in that web browser
+  6. For authenticated API access use the `ssl/owner/cert.pem` HTTPS client certificate.
+     If you are running Linux with user other than `root`, you might need to fix the certificate permissions because Docker bind mounts are owned by `root` by default. For example:
+     ```shell
+     sudo setfacl -m u:$(whoami):r ./ssl/owner/*
+     ```
+  7. Open **https://localhost:4443/** in the web browser or use `curl` for API access, for example:
+     ```shell
+     curl -k -E ./ssl/owner/cert.pem:<your cert password> -H "Accept: text/turtle" 'https://localhost:4443/'
+     ```
 
   ### Notes
 
@@ -87,11 +99,11 @@ It takes a few clicks and filling out a form to install the product into your ow
     * If this option does not appear in Chrome (as observed on some MacOS), you can open `chrome://flags/#allow-insecure-localhost`, switch `Allow invalid certificates for resources loaded from localhost` to `Enabled` and restart Chrome
   * `.env_sample` and `.env` files might be invisible in MacOS Finder which hides filenames starting with a dot. You should be able to [create it using Terminal](https://stackoverflow.com/questions/5891365/mac-os-x-doesnt-allow-to-name-files-starting-with-a-dot-how-do-i-name-the-hta) however.
   * On Linux your user may need to be a member of the `docker` group. Add it using
-  ```
+  ```shell
   sudo usermod -aG docker ${USER}
   ```
   and re-login with your user. An alternative, but not recommended, is to run
-  ```
+  ```shell
   sudo docker-compose up
   ```
 </details>
@@ -152,7 +164,7 @@ The options are described in more detail in the [configuration documentation](ht
   ## Reset
 
   If you need to start fresh and wipe the existing setup (e.g. after configuring a new base URI), you can do that using
-  ```
+  ```shell
   sudo rm -rf data uploads && docker-compose down -v
   ```
 
@@ -169,11 +181,15 @@ _:warning: This will **remove the persisted data and files** as well as Docker v
 
 LinkedDataHub CLI wraps the HTTP API into a set of shell scripts with convenient parameters. The scripts can be used for testing, automation, scheduled execution and such. It is usually much quicker to perform actions using CLI rather than the user interface, as well as easier to reproduce.
 
-The scripts can be found in the [`scripts`](https://github.com/AtomGraph/LinkedDataHub/tree/master/scripts) subfolder.
+The scripts can be found in the [`bin`](https://github.com/AtomGraph/LinkedDataHub/tree/master/bin) subfolder. In order to use them, add the `bin` folder and its subfolders to the `$PATH`. For example:
+
+```shell
+export PATH="$(find bin -type d -exec realpath {} \; | tr '\n' ':')$PATH"
+```
 
 _:warning: The CLI scripts internally use [Jena's CLI commands](https://jena.apache.org/documentation/tools/). Set up the Jena environment before running the scripts._
 
-An environment variable `JENA_HOME` is used by all the command line tools to configure the class path automatically for you. You can set this up as follows:
+The environment variable `JENA_HOME` is used by all the command line tools to configure the class path automatically for you. You can set this up as follows:
 
 **On Linux / Mac**
 
@@ -189,11 +205,7 @@ An environment variable `JENA_HOME` is used by all the command line tools to con
 
 ### [Demo apps](https://github.com/AtomGraph/LinkedDataHub-Apps)
 
-These demo applications can be installed into a LinkedDataHub instance using the provided CLI scripts.
-
-_:warning: Before running app installation scripts that use LinkedDataHub's CLI scripts, set the `SCRIPT_ROOT` environmental variable to the [`scripts`](https://github.com/AtomGraph/LinkedDataHub/tree/master/scripts) subfolder of your LinkedDataHub fork or clone._ For example:
-
-    export SCRIPT_ROOT="/c/Users/namedgraph/WebRoot/AtomGraph/LinkedDataHub/scripts"
+These demo applications can be installed into a LinkedDataHub instance using `make install`. You will need to provide the path to your WebID certificate as well as its password.
 
 ## How to get involved