diff --git a/.gitignore b/.gitignore index 57630191..38c0a89b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ **/.DS_Store **/*.rpm **/*.deb -**/*-registry.yml \ No newline at end of file +**/*-registry.yml +CLAUDE.md +.claude +testdata \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c74eddb0..f231005a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,7 +91,7 @@ SHELL ["/bin/bash", "-c"] RUN mkdir -p \ ${CSGHUB_HOME}/{LICENSES,bin} \ - ${CSGHUB_EMBEDDED}/{bin,lib,sv} \ + ${CSGHUB_EMBEDDED}/{bin,lib,sv,etc} \ ${CSGHUB_SRV_HOME}/{registry,nats,temporal,temporal_ui,casdoor,dnsmasq,consul,server,portal,prometheus,loki}/bin ## Install Runit Service Daemon @@ -128,33 +128,37 @@ COPY --from=gitaly /usr/local/. ${CSGHUB_SRV_HOME}/gitaly/ COPY --from=gitlab-shell /srv/gitlab-shell/. ${CSGHUB_SRV_HOME}/gitlab_shell/ ## Install Temporal & Temporal-ui -COPY --from=temporal ${CSGHUB_HOME}/etc/temporal/. ${CSGHUB_HOME}/etc/temporal/ +COPY --from=temporal ${CSGHUB_HOME}/etc/temporal/. ${CSGHUB_EMBEDDED}/etc/temporal/ COPY --from=temporal ${CSGHUB_SRV_HOME}/temporal/. ${CSGHUB_SRV_HOME}/temporal/ -COPY --from=temporal ${CSGHUB_HOME}/etc/temporal_ui/. ${CSGHUB_HOME}/etc/temporal_ui/ +COPY --from=temporal ${CSGHUB_HOME}/etc/temporal_ui/. ${CSGHUB_EMBEDDED}/etc/temporal_ui/ COPY --from=temporal ${CSGHUB_SRV_HOME}/temporal_ui/. ${CSGHUB_SRV_HOME}/temporal_ui/ # Using 8182 as temporal-ui default listen port -RUN sed -i 's/8080/8182/g' ${CSGHUB_HOME}/etc/temporal_ui/config-template.yaml +RUN sed -i 's/8080/8182/g' ${CSGHUB_EMBEDDED}/etc/temporal_ui/config-template.yaml + +# Remap temporal internal paths from component image to new embedded location +RUN find ${CSGHUB_EMBEDDED}/etc/temporal -type f -exec sed -i 's|/opt/csghub/etc/temporal|/opt/csghub/embedded/etc/temporal|g' {} \; ## Install NATS COPY --from=nats /nats-server ${CSGHUB_SRV_HOME}/nats/bin/ ## Install Casdoor COPY --from=casdoor /server ${CSGHUB_SRV_HOME}/casdoor/bin/casdoor -COPY --from=casdoor /web ${CSGHUB_HOME}/etc/casdoor/web +COPY --from=casdoor /web ${CSGHUB_EMBEDDED}/etc/casdoor/web ## Install Nginx -COPY --from=nginx /opt/csghub/. /opt/csghub/ -COPY ./opt/csghub/etc/nginx/. /opt/csghub/etc/nginx/ +COPY --from=nginx /opt/csghub/embedded/sv/nginx/. /opt/csghub/embedded/sv/nginx/ +COPY --from=nginx /opt/csghub/etc/nginx/. /opt/csghub/embedded/etc/nginx/ +COPY ./opt/csghub/embedded/etc/nginx/. /opt/csghub/embedded/etc/nginx/ ## Install csghub-server ENV GIN_MODE=release COPY --from=server /starhub-bin/starhub ${CSGHUB_SRV_HOME}/server/bin/csghub-server -COPY --from=server /starhub-bin/. ${CSGHUB_HOME}/etc/server/ +COPY --from=server /starhub-bin/. ${CSGHUB_EMBEDDED}/etc/server/ COPY --from=server /root/.duckdb ${CSGHUB_SRV_HOME}/server/.duckdb -RUN rm ${CSGHUB_HOME}/etc/server/starhub +RUN rm ${CSGHUB_EMBEDDED}/etc/server/starhub ## Install csghub-portal COPY --from=portal /myapp/csghub-portal ${CSGHUB_SRV_HOME}/portal/bin/ @@ -275,7 +279,7 @@ RUN chmod +x -R /opt/csghub/bin && \ chmod +x -R /opt/csghub/embedded/sv/**/templates && \ chmod +x -R /scripts && \ ln -s /opt/csghub/bin/* /usr/bin/ && \ - find /opt/csghub/etc/ -type f -name "*.sh" -exec chmod +x {} \; + find /opt/csghub/embedded/etc/ -type f -name "*.sh" -exec chmod +x {} \; EXPOSE 80 443 2222 5000 8000 9000 9001 ENTRYPOINT ["/scripts/entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md index 2986cfdf..a0d15946 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Omnibus-CSGHub 是 OpenCSG CSGHub 的一键式 Docker Compose 部署方案,让 *Contains all configurable parameters, with minimal modifications typically required under default settings.* -- 默认参数文件 `/opt/csghub/etc/csghub/default.yaml` +- 默认参数文件 `/opt/csghub/embedded/etc/csghub/default.yaml` 定义了所有参数的默认值。 diff --git a/dockerfiles/toolbox/Dockerfile_toolbox b/dockerfiles/toolbox/Dockerfile_toolbox index cac0bb3e..75f3206a 100644 --- a/dockerfiles/toolbox/Dockerfile_toolbox +++ b/dockerfiles/toolbox/Dockerfile_toolbox @@ -45,17 +45,14 @@ RUN if grep -q -i -E 'ubuntu|debian' /etc/os-release; then \ ## Generate ssh key pairs RUN ssh-keygen -A -## Install gompalte -ARG GOMPLATE_VERSION=v4.3.2 +## Install kubectl ARG KUBECTL_VERSION=v1.33.0 ARG TARGETPLATFORM RUN case ${TARGETPLATFORM} in \ "linux/amd64") \ - wget --no-check-certificate -O ${CSGHUB_EMBEDDED}/bin/gomplate https://github.com/hairyhenderson/gomplate/releases/download/${GOMPLATE_VERSION}/gomplate_linux-amd64 && \ wget --no-check-certificate -O ${CSGHUB_EMBEDDED}/bin/kubectl https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ ;; \ "linux/arm64") \ - wget --no-check-certificate -O ${CSGHUB_EMBEDDED}/bin/gomplate https://github.com/hairyhenderson/gomplate/releases/download/${GOMPLATE_VERSION}/gomplate_linux-arm64 && \ wget --no-check-certificate -O ${CSGHUB_EMBEDDED}/bin/kubectl https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/arm64/kubectl \ ;; \ *) \ diff --git a/dockerfiles/toolbox/version-manifests.json b/dockerfiles/toolbox/version-manifests.json index 5f31d51f..d61dce91 100644 --- a/dockerfiles/toolbox/version-manifests.json +++ b/dockerfiles/toolbox/version-manifests.json @@ -3,11 +3,10 @@ { "name": "toolbox", "description": "Containerized tooling for system operations", - "version": "1.3.2", + "version": "1.3.3", "environments": { "OS_RELEASE": "ubuntu:22.04", - "TOOLBOX_VERSION": "1.3.2", - "GOMPLATE_VERSION": "v4.3.2", + "TOOLBOX_VERSION": "1.3.3", "KUBECTL_VERSION": "v1.33.0", "DNSMASQ_VERSION": "2.91" } diff --git a/ee.Dockerfile b/ee.Dockerfile index 4715c35d..80284462 100644 --- a/ee.Dockerfile +++ b/ee.Dockerfile @@ -109,7 +109,7 @@ SHELL ["/bin/bash", "-c"] RUN mkdir -p \ ${CSGHUB_HOME}/{LICENSES,bin} \ - ${CSGHUB_EMBEDDED}/{bin,lib,sv} \ + ${CSGHUB_EMBEDDED}/{bin,lib,sv,etc} \ ${CSGHUB_SRV_HOME}/{registry,nats,temporal,temporal_ui,casdoor,dnsmasq,consul,server,portal,prometheus,loki,xnet}/bin ## Install Runit Service Daemon @@ -146,9 +146,9 @@ COPY --from=gitaly /usr/local/. ${CSGHUB_SRV_HOME}/gitaly/ COPY --from=gitlab-shell /srv/gitlab-shell/. ${CSGHUB_SRV_HOME}/gitlab_shell/ ## Install Temporal & Temporal-ui -COPY --from=temporal ${CSGHUB_HOME}/etc/temporal/. ${CSGHUB_HOME}/etc/temporal/ +COPY --from=temporal ${CSGHUB_HOME}/etc/temporal/. ${CSGHUB_EMBEDDED}/etc/temporal/ COPY --from=temporal ${CSGHUB_SRV_HOME}/temporal/. ${CSGHUB_SRV_HOME}/temporal/ -COPY --from=temporal ${CSGHUB_HOME}/etc/temporal_ui/. ${CSGHUB_HOME}/etc/temporal_ui/ +COPY --from=temporal ${CSGHUB_HOME}/etc/temporal_ui/. ${CSGHUB_EMBEDDED}/etc/temporal_ui/ COPY --from=temporal ${CSGHUB_SRV_HOME}/temporal_ui/. ${CSGHUB_SRV_HOME}/temporal_ui/ ## Install NATS @@ -156,20 +156,20 @@ COPY --from=nats /nats-server ${CSGHUB_SRV_HOME}/nats/bin/ ## Install Casdoor COPY --from=casdoor /server ${CSGHUB_SRV_HOME}/casdoor/bin/casdoor -COPY --from=casdoor /web ${CSGHUB_HOME}/etc/casdoor/web +COPY --from=casdoor /web ${CSGHUB_EMBEDDED}/etc/casdoor/web ## Install Nginx -COPY --from=nginx /opt/csghub/. /opt/csghub/ -#COPY ./opt/csghub/etc/nginx/. /opt/csghub/etc/nginx/ +COPY --from=nginx /opt/csghub/embedded/sv/nginx/. /opt/csghub/embedded/sv/nginx/ +COPY --from=nginx /opt/csghub/etc/nginx/. /opt/csghub/embedded/etc/nginx/ ## Install csghub-server ENV GIN_MODE=release COPY --from=server /starhub-bin/starhub ${CSGHUB_SRV_HOME}/server/bin/csghub-server -COPY --from=server /starhub-bin/. ${CSGHUB_HOME}/etc/server/ +COPY --from=server /starhub-bin/. ${CSGHUB_EMBEDDED}/etc/server/ COPY --from=server /root/.duckdb ${CSGHUB_SRV_HOME}/server/.duckdb -RUN rm ${CSGHUB_HOME}/etc/server/starhub +RUN rm ${CSGHUB_EMBEDDED}/etc/server/starhub ## Install csghub-portal COPY --from=portal /myapp/csghub-portal ${CSGHUB_SRV_HOME}/portal/bin/ @@ -187,16 +187,19 @@ COPY --from=loki /usr/bin/loki ${CSGHUB_SRV_HOME}/loki/bin/ ## Install csgship-web COPY --from=csgship /code/. ${CSGHUB_SRV_HOME}/web/ COPY --from=billing /app/. ${CSGHUB_SRV_HOME}/billing/bin/ -COPY --from=frontend /usr/share/nginx/html ${CSGHUB_HOME}/etc/nginx/html +COPY --from=frontend /usr/share/nginx/html ${CSGHUB_EMBEDDED}/etc/nginx/html COPY --from=agentic /code/. ${CSGHUB_SRV_HOME}/agentic/ # Using 8182 as temporal-ui default listen port -RUN sed -i 's/8080/8182/g' ${CSGHUB_HOME}/etc/temporal_ui/config-template.yaml && \ +RUN sed -i 's/8080/8182/g' ${CSGHUB_EMBEDDED}/etc/temporal_ui/config-template.yaml && \ sed -i -e 's/:8000/:8183/g' \ -e 's|/code/logs/gunicorn.access.log|/dev/stdout|g' \ -e 's|/code/|/opt/csghub/embedded/sv/web/|g' \ ${CSGHUB_EMBEDDED}/sv/web/project/{gunicorn_config.py,uwsgi.ini} +# Remap temporal internal paths from component image to new embedded location +RUN find ${CSGHUB_EMBEDDED}/etc/temporal -type f -exec sed -i 's|/opt/csghub/etc/temporal|/opt/csghub/embedded/etc/temporal|g' {} \; + ENV PATH=$PATH:/opt/csghub/embedded/bin RUN if grep -q -i -E 'ubuntu|debian' /etc/os-release; then \ apt update && \ @@ -303,7 +306,7 @@ RUN chmod +x -R /opt/csghub/bin && \ chmod +x -R /opt/csghub/embedded/sv/**/templates && \ chmod +x -R /scripts && \ ln -s /opt/csghub/bin/* /usr/bin/ && \ - find /opt/csghub/etc/ -type f -name "*.sh" -exec chmod +x {} \; + find /opt/csghub/embedded/etc/ -type f -name "*.sh" -exec chmod +x {} \; EXPOSE 80 443 2222 5000 8000 9000 9001 ENTRYPOINT ["/scripts/entrypoint.sh"] \ No newline at end of file diff --git a/ee/opt/csghub/embedded/etc/billing/config.yaml.sample b/ee/opt/csghub/embedded/etc/billing/config.yaml.sample new file mode 100644 index 00000000..4d18df87 --- /dev/null +++ b/ee/opt/csghub/embedded/etc/billing/config.yaml.sample @@ -0,0 +1,15 @@ +{{- $natsUser := .nats.auth.user }} +{{- $natsPassword := .nats.auth.password | default (pbkdf2 $natsUser "opencsg" 2048 8) }} +{{- $webNats := .web.nats }} +nats: {{ $webNats.url | default (printf "nats://%s:%s@localhost:4222" $natsUser $natsPassword) }} +pub: + addr: {{ .billing.listen }} + key: '' +sub: + streams: + - name: accountingNotifyStream + consumers: + - durable_name: starship-billing-durable-consumer + webhook: + url: http://127.0.0.1:8001/api/v1/platforms/billing-events/ + key: 'key' diff --git a/ee/opt/csghub/etc/csghub/default.yaml b/ee/opt/csghub/embedded/etc/csghub/default.yaml similarity index 99% rename from ee/opt/csghub/etc/csghub/default.yaml rename to ee/opt/csghub/embedded/etc/csghub/default.yaml index 80db8bad..866a149f 100644 --- a/ee/opt/csghub/etc/csghub/default.yaml +++ b/ee/opt/csghub/embedded/etc/csghub/default.yaml @@ -80,7 +80,7 @@ server: ## Casdoor (authentication) Integration casdoor: endpoint: "" ## Casdoor endpoint - certificate: "/opt/csghub/etc/casdoor/token_jwt_key.pem" ## SSL cert path + certificate: "/opt/csghub/embedded/etc/casdoor/token_jwt_key.pem" ## SSL cert path organization_name: "OpenCSG" ## Organization name in Casdoor application_name: "CSGHub" ## App name in Casdoor diff --git a/ee/opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample b/ee/opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample similarity index 99% rename from ee/opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample rename to ee/opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample index e1d316ae..e81a8431 100644 --- a/ee/opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample +++ b/ee/opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample @@ -74,7 +74,7 @@ csghub: ## Casdoor (authentication) Integration # casdoor: # endpoint: "http://csghub.example.com:8000" ## Casdoor endpoint - # certificate: "/opt/csghub/etc/casdoor/token_jwt_key.pem" ## SSL cert path + # certificate: "/opt/csghub/embedded/etc/casdoor/token_jwt_key.pem" ## SSL cert path # organization_name: "OpenCSG" ## Organization name in Casdoor # application_name: "CSGHub" ## App name in Casdoor diff --git a/ee/opt/csghub/embedded/etc/nginx/html/config.js.sample b/ee/opt/csghub/embedded/etc/nginx/html/config.js.sample new file mode 100644 index 00000000..68827ddd --- /dev/null +++ b/ee/opt/csghub/embedded/etc/nginx/html/config.js.sample @@ -0,0 +1,7 @@ +{{- $csghub := resolveUrl .csghub.external_url }} +{{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} +window.envConfig = { + VITE_APP_BASE_WEB: {{ (resolveEndpoint $csghub.scheme $csghub.host .csgship.listen.frontend "127.0.0.1:8001").url | squote }}, + VITE_APP_BASE_API: {{ (resolveEndpoint $csghub.scheme $csghub.host .csgship.listen.api "127.0.0.1:8002").url | squote }}, + VITE_APP_LOGIN_URL: {{ printf "%s://%s:%s/login/oauth/authorize?client_id=%s&response_type=code&redirect_uri=%s/api/v1/account/casdoor/login/callback&scope=profile&state=casdoor" $csghub.scheme $csghub.host $casdoorEp.port (generateClientId "CSGShip") $csghub.url | squote }} +}; diff --git a/ee/opt/csghub/etc/nginx/nginx.conf.sample b/ee/opt/csghub/embedded/etc/nginx/nginx.conf.sample similarity index 86% rename from ee/opt/csghub/etc/nginx/nginx.conf.sample rename to ee/opt/csghub/embedded/etc/nginx/nginx.conf.sample index 9061584f..2d9e1d2b 100644 --- a/ee/opt/csghub/etc/nginx/nginx.conf.sample +++ b/ee/opt/csghub/embedded/etc/nginx/nginx.conf.sample @@ -11,7 +11,7 @@ events { use epoll; # Use epoll for scalable I/O } -{{ $nginx := (datasource "config").nginx }} +{{- $csghub := resolveUrl .csghub.external_url }} # HTTP block configuration http { # Disable server version token in responses @@ -96,20 +96,16 @@ http { "~^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" 1; } - {{- if $nginx.rate_limit.enable }} - limit_req_zone $binary_remote_addr zone=global:10m rate={{ $nginx.rate_limit.request }}; + {{- if .nginx.rate_limit.enable }} + limit_req_zone $binary_remote_addr zone=global:10m rate={{ .nginx.rate_limit.request }}; limit_conn_zone $binary_remote_addr zone=addr:10m; {{- end }} - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} - {{- $server := (datasource "config").server -}} - {{- $nginx := (datasource "config").nginx -}} - {{- $dataDir := $nginx.data }} + {{- $dataDir := .nginx.data }} - {{- $csghubCert := $nginx.ssl_certificate | default (printf "/etc/csghub/ssl/%s.crt" $csghub.host) | quote }} - {{- $csghubCertKey := $nginx.ssl_certificate_key | default (printf "/etc/csghub/ssl/%s.key" $csghub.host) | quote }} - - {{- if eq $csghub.scheme "https" -}} + {{- $csghubCert := .nginx.ssl_certificate | default (printf "/etc/csghub/ssl/%s.crt" $csghub.host) | quote }} + {{- $csghubCertKey := .nginx.ssl_certificate_key | default (printf "/etc/csghub/ssl/%s.key" $csghub.host) | quote }} + {{- if eq $csghub.scheme "https" }} # Main server configurations server { listen 80; @@ -135,9 +131,9 @@ http { ssl_certificate_key {{ $csghubCertKey }}; {{- end }} - {{- if $nginx.rate_limit.enable }} - limit_req zone=global burst={{ $nginx.rate_limit.burst }}; - limit_conn addr {{ $nginx.rate_limit.conn }}; + {{- if .nginx.rate_limit.enable }} + limit_req zone=global burst={{ .nginx.rate_limit.burst }}; + limit_conn addr {{ .nginx.rate_limit.conn }}; {{- end }} # Default route @@ -171,10 +167,9 @@ http { } # xnet - {{ $xnet := (datasource "config").xnet -}} location ~ ^/xnet/ { rewrite ^/xnet/(.*) /$1 break; - proxy_pass http://127.0.0.1:{{ $xnet.listen_port }}; + proxy_pass http://127.0.0.1:{{ .xnet.listen_port }}; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -192,10 +187,9 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } - {{- $temporalUI := (datasource "config").temporal_ui -}} - {{ if $temporalUI.enable }} + {{- if .temporal_ui.enable }} # Temporal UI route with basic auth - location {{ $temporalUI.public_path }} { + location {{ .temporal_ui.public_path }} { proxy_pass http://127.0.0.1:8182; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -236,8 +230,7 @@ http { proxy_read_timeout 900; } - {{- $runner := (datasource "config").runner -}} - {{- if and (not $runner.use_public_domain) (regexp.Match `^[a-zA-Z0-9.-]+$` $csghub.host) }} + {{- if and (not .runner.use_public_domain) (regexMatch `^[a-zA-Z0-9.-]+$` $csghub.host) }} # Endpoint route location /endpoint/ { proxy_pass http://127.0.0.1:8083; @@ -272,9 +265,9 @@ http { } } - {{ if and $runner.use_public_domain - (regexp.Match `^[a-zA-Z0-9.-]+$` $csghub.host) - (not (regexp.Match `^[0-9.]+$` $csghub.host)) -}} + {{ if and .runner.use_public_domain + (regexMatch `^[a-zA-Z0-9.-]+$` $csghub.host) + (not (regexMatch `^[0-9.]+$` $csghub.host)) -}} server { {{- if eq $csghub.scheme "https" }} listen 443 ssl; @@ -288,9 +281,9 @@ http { ssl_certificate_key {{ $csghubCertKey }}; {{- end }} - {{- if $nginx.rate_limit.enable }} - limit_req zone=global burst={{ $nginx.rate_limit.burst }}; - limit_conn addr {{ $nginx.rate_limit.conn }}; + {{- if .nginx.rate_limit.enable }} + limit_req zone=global burst={{ .nginx.rate_limit.burst }}; + limit_conn addr {{ .nginx.rate_limit.conn }}; {{- end }} # Endpoint route @@ -318,23 +311,23 @@ http { } {{- end }} - {{ $casdoor := tmpl.Exec "config.casdoor.conn" . | data.YAML -}} # Casdoor service configuration + {{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} server { {{- if eq $csghub.scheme "https" }} - listen {{ $casdoor.port }} ssl; - server_name {{ $csghub.host }}; + listen {{ $casdoorEp.port }} ssl; + server_name {{ $casdoorEp.host }}; ssl_certificate {{ $csghubCert }}; ssl_certificate_key {{ $csghubCertKey }}; {{- else }} - listen {{ $casdoor.port }}; - server_name {{ $csghub.host }}; + listen {{ $casdoorEp.port }}; + server_name {{ $casdoorEp.host }}; {{- end }} - {{- if $nginx.rate_limit.enable }} - limit_req zone=global burst={{ $nginx.rate_limit.burst }}; - limit_conn addr {{ $nginx.rate_limit.conn }}; + {{- if .nginx.rate_limit.enable }} + limit_req zone=global burst={{ .nginx.rate_limit.burst }}; + limit_conn addr {{ .nginx.rate_limit.conn }}; {{- end }} location / { @@ -415,9 +408,8 @@ http { } } - {{ $csgship := (datasource "config").csgship -}} - {{- if $csgship.enable }} - {{- $csgshipFrontend := tmpl.Exec "config.csgship.frontend" . | data.YAML -}} + {{- if .csgship.enable }} + {{- $csgshipFrontend := resolveEndpoint $csghub.scheme $csghub.host .csgship.listen.frontend "127.0.0.1:8001" }} server { {{- if eq $csghub.scheme "https" }} listen {{ $csgshipFrontend.port }} ssl; @@ -430,7 +422,7 @@ http { server_name {{ $csgshipFrontend.host }}; {{- end }} - {{ $dataDir := (datasource "config").nginx.data | default "/var/opt/csghub/nginx" }} + {{- $dataDir := .nginx.data | default "/var/opt/csghub/nginx" }} location / { root {{ $dataDir }}/html; index index.html; @@ -438,22 +430,21 @@ http { } } - {{ $csgshipApi := tmpl.Exec "config.csgship.api" . | data.YAML -}} + {{- $csgshipAPI := resolveEndpoint $csghub.scheme $csghub.host .csgship.listen.api "127.0.0.1:8002" }} server { {{- if eq $csghub.scheme "https" }} - listen {{ $csgshipApi.port }} ssl; - server_name {{ $csgshipApi.host }}; + listen {{ $csgshipAPI.port }} ssl; + server_name {{ $csgshipAPI.host }}; ssl_certificate {{ $csghubCert }}; ssl_certificate_key {{ $csghubCertKey }}; {{- else }} - listen {{ $csgshipApi.port }}; - server_name {{ $csgshipApi.host }}; + listen {{ $csgshipAPI.port }}; + server_name {{ $csgshipAPI.host }}; {{- end }} location /agentic { - {{- $agentic := (datasource "config").agentic }} - proxy_pass http://{{ $agentic.listen }}; + proxy_pass http://{{ .agentic.listen }}; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -496,4 +487,4 @@ http { } } {{- end }} -} \ No newline at end of file +} diff --git a/ee/opt/csghub/embedded/etc/server/config.toml.sample b/ee/opt/csghub/embedded/etc/server/config.toml.sample new file mode 100644 index 00000000..a5457244 --- /dev/null +++ b/ee/opt/csghub/embedded/etc/server/config.toml.sample @@ -0,0 +1,328 @@ +{{- $csghub := resolveUrl .csghub.external_url }} +saas = {{ .server.saas }} +oversea = false +instance_id = "" +enable_swagger = {{ .server.enable_swagger }} +enable_https = {{ .server.enable_https }} +api_token = {{ deriveToken | quote }} +docs_host = {{ printf "%s://%s:6636" $csghub.scheme $csghub.host | quote }} +server_failure_redirect_url = {{ printf "%s/errors/server-error" $csghub.url | quote }} + +[api_server] +port = {{ .server.listen_port }} +public_domain = {{ $csghub.url | quote }} +{{- $shell_port := .gitlab_shell.ssh_port }} +{{- if eq $shell_port 22 }} +ssh_domain = {{ printf "git@%s:%d" (toString $csghub.host) (int64 $shell_port) | quote }} +{{- else }} +ssh_domain = {{ printf "ssh://git@%s:%d" (toString $csghub.host) (int64 $shell_port) | quote }} +{{- end }} + +[mirror] +url = {{ printf "http://localhost:%d" .mirror_repo.listen_port | quote }} +token = "" +port = {{ .mirror_repo.listen_port }} +{{- if .server.saas }} +remote = true +{{- else }} +remote = false +{{- end }} +session_secret_key = {{ persistentSeed | sha1sum | quote }} +worker_number = {{ .mirror_repo.worker_number }} +lfs_concurrency = {{ .mirror_lfs.lfs_concurrency }} +part_size = {{ .mirror_lfs.part_size }} + +{{- $postgresql := .server.postgresql }} +{{- $pgUser := $postgresql.user }} +{{- $pgPassword := $postgresql.password | default (pbkdf2 $pgUser "opencsg" 2048 8) }} + +[database] +driver = "pg" +dsn = {{ printf "postgresql://%s:%s@%s:%d/%s?sslmode=disable" (toString $pgUser) (toString $pgPassword) (toString $postgresql.host) (int64 $postgresql.port) (toString $postgresql.name) | quote }} +timezone = {{ $postgresql.timezone | quote }} + +{{- $redis := .server.redis }} +{{- $redisUser := $redis.auth.user }} +{{- $redisPassword := $redis.auth.password | default (pbkdf2 $redisUser "opencsg" 2048 8) }} + +[redis] +endpoint = {{ $redis.endpoint | quote }} +max_retries = {{ $redis.max_retries }} +min_idle_connections = {{ $redis.min_idle_connections }} +user = {{ $redisUser | quote }} +password = {{ $redisPassword | quote }} +sentinel_mode = {{ $redis.sentinel.mode }} +sentinel_endpoint = {{ $redis.sentinel.endpoint | quote }} +sentinel_master = {{ $redis.sentinel.master | quote }} + +[git_server] +type = "gitaly" + +{{- $gitaly := .server.gitaly }} +{{- $gitalyToken := $gitaly.token | default (pbkdf2 "gitaly" "opencsg" 2048 8) }} + +[gitaly_server] +address = {{ $gitaly.address | quote }} +storge = {{ $gitaly.storage | quote }} +token = {{ $gitalyToken | quote }} +jwt_secret ={{ $gitaly.secret | default (persistentSeed | sha256sum) | quote }} + +[frontend] +url = {{ $csghub.url | quote }} + +{{- $s3 := .server.s3 }} +{{- $accessKeyId := $s3.access_key }} +{{- $secretKey := $s3.secret_key | default (pbkdf2 $accessKeyId "opencsg" 2048 8) }} + +[s3] +access_key_id = {{ $accessKeyId | quote }} +access_key_secret = {{ $secretKey | quote }} +region = {{ $s3.region | quote }} +endpoint = {{ $s3.endpoint | default (printf "%s:9000" $csghub.host) | quote }} +internal_endpoint = {{ $s3.internal_endpoint | quote }} +bucket = {{ $s3.bucket | quote }} +enable_ssl = {{ $s3.secure }} +{{- if $s3.path_style }} +bucket_lookup = "path" +{{- else }} +bucket_lookup = "auto" +{{- end }} + +[jwt] +signing_key = {{ $gitaly.secret | default (persistentSeed | sha256sum) | quote }} +valid_hour = 24 + +[space] +internal_root_domain = "spaces.app.internal:8083" +{{- if .runner.use_public_domain }} +public_root_domain = {{ printf "%s:%v" $csghub.host $csghub.port | quote }} +{{- else }} +public_root_domain = "" +{{- end }} +session_secret_key = {{ persistentSeed | sha1sum | quote }} +pypi_index_url = {{ .runner.pip_index_url | quote }} + +{{- $model := .runner.model }} + +[model] +deploy_timeout_in_min = {{ $model.deploy_timeout }} +download_endpoint = {{ $csghub.url | quote }} +docker_reg_base = {{ $model.registry | quote }} +nim_docker_secret_name = "ngc-secret" +nim_ngc_secret_name = "nvidia-nim-secrets" + +[event] +sync_interval = {{ .mirror_repo.sync_interval }} + +{{- $serverCasdoor := .server.casdoor }} +{{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} +[casdoor] +client_id = {{ generateClientId "CSGHub" | quote }} +client_secret = {{ generateClientSecret "CSGHub" | quote }} +endpoint = {{ $serverCasdoor.endpoint | default ($casdoorEp.url) | quote }} +certificate = {{ $serverCasdoor.certificate | quote }} +organization_name = {{ $serverCasdoor.organization_name | quote }} +application_name = {{ $serverCasdoor.application_name | quote }} + +{{- $natsUser := .nats.auth.user }} +{{- $natsPassword := .nats.auth.password | default (pbkdf2 $natsUser "opencsg" 2048 8) }} +{{- $serverNats := .server.nats }} + +[nats] +url = {{ $serverNats.url | default (printf "nats://%s:%s@localhost:4222" (toString $natsUser) (toString $natsPassword)) | quote }} +msg_fetch_timeout_in_sec = {{ $serverNats.msg_fetch_timeout_in_sec }} + +{{- $actParts := .accounting.listen | toString | splitList ":" }} +{{- $actHost := index $actParts 0 }} +{{- $actPort := "" }} +{{- if gt (len $actParts) 1 }} + {{- $actPort = index $actParts 1 }} +{{- else }} + {{- $actPort = "8086" }} +{{- end }} + +[accounting] +host = {{ printf "http://%s" $actHost | quote }} +port = {{ $actPort }} + +{{- $userParts := .user.listen | splitList ":" }} +{{- $userHost := index $userParts 0 }} +{{- $userPort := "" }} +{{- if gt (len $userParts) 1 }} + {{- $userPort = index $userParts 1 }} +{{- else }} + {{- $actPort = "8088" }} +{{- end }} + +[user] +host = {{ printf "http://%s" $userHost | quote }} +port = {{ $userPort }} +signin_success_redirect_url = {{ printf "%s/server/callback" $csghub.url | quote }} + +{{- $multi_sync := .server.multi_sync }} + +[multi_sync] +enabled = {{ $multi_sync.enabled }} +saas_api_domain = "https://hub.opencsg.com" +saas_sync_domain = "https://sync.opencsg.com" + +{{- $telemetry := .server.telemetry }} + +[telemetry] +enable = {{ $telemetry.enabled | default true }} +report_url = "http://hub.opencsg.com/api/v1/telemetry" + +[auto_clean] +instance = false + +{{- $dataset := .server.dataset }} + +[dataset] +prompt_max_jsonl_file_size = {{ $dataset.prompt_max_jsonl_file_size }} + +{{- $dataflow := .server.dataflow }} +{{- $dfUrl := parseURL $dataflow.address }} +{{- $dfUrlParts := $dfUrl.Host | splitList ":" }} +{{- $dfHost := index $dfUrlParts 0 }} +{{- $dfPort := "" }} +{{- if gt (len $dfUrlParts) 1 }} + {{- $dfPort = index $dfUrlParts 1 }} +{{- else }} + {{- if eq $dfUrl.Scheme "https" }} + {{- $dfPort = "443" }} + {{- else }} + {{- $dfPort = "80" }} + {{- end }} +{{- end }} + +[dataflow] +host = {{ printf "%s://%s" $dfUrl.Scheme $dfHost | quote }} +port = {{ $dfPort }} + +{{- if .moderation.enable }} +{{- $moderationAddress := .moderation.address }} +{{- $moderationUrl := parseURL $moderationAddress }} +{{- $moderationParts := $moderationUrl.Host | splitList ":" }} +{{- $moderationHost := index $moderationParts 0 | default "127.0.0.1" }} +{{- $moderationPort := "" }} +{{- if gt (len $moderationParts) 1 }} + {{- $moderationPort = index $moderationParts 1 }} +{{- else }} + {{- $moderationPort = "8089" }} +{{- end }} + +[moderation] +host = {{ printf "%s://%s" $moderationUrl.Scheme $moderationHost | quote }} +port = {{ $moderationPort }} +encoded_sensitive_words = {{ .moderation.encoded_sensitive_words | quote }} + +{{- $sensitive := .moderation.sensitive_check }} + +[sensitive_check] +enable = true +access_key_id = {{ $sensitive.access_key_id | quote }} +access_key_secret = {{ $sensitive.access_key_secret | quote }} +region = {{ $sensitive.region | quote }} +endpoint = {{ $sensitive.endpoint | quote }} +enable_ssl = {{ $sensitive.enable_ssl }} +{{- end }} + +{{- $workflow := .server.workflow }} + +[workflow] +endpoint = {{ $workflow.endpoint | quote }} + +[cron_job] +sync_as_client_cron_expression = {{ $multi_sync.sync_as_client_cron_expression | quote }} +calc_recom_score_cron_expression = {{ $multi_sync.calc_recom_score_cron_expression | quote }} + +{{- $proxy := .server.proxy }} + +[proxy] +hosts = [{{- range $k, $v := $proxy.hosts }}{{if $k}}, {{end}}{{$v | quote }}{{- end }}] + +{{- $instrumentation := .server.instrumentation }} + +[instrumentation] +otlp_logging = {{ $instrumentation.otlp_logging }} +otlp_endpoint = {{ $instrumentation.otlp_endpoint | quote }} + +{{- $git := .server.git }} + +[git] +operation_timeout = {{ $git.operation_timeout }} +{{- if .praefect.enable }} +check_file_size_enabled = false +{{- else }} +check_file_size_enabled = {{ $git.check_file_size_enabled }} +{{- end }} +max_un_lfs_file_size = {{ $git.max_un_lfs_file_size }} +skip_lfs_file_validation = {{ $git.skip_lfs_file_validation }} + +{{- if .notifier.enable }} +{{- $notifyListen := .notifier.listen }} +{{- $notifyParts := $notifyListen | splitList ":" }} +{{- $notifyHost := index $notifyParts 0 | default "127.0.0.1" }} +{{- $notifyPort := "8095" }} +{{- if eq (len $notifyParts) 2 }} + {{- $notifyPort = index $notifyParts 1 }} +{{- end }} + +[notification] +host = {{ printf "http://%s" $notifyHost | quote }} +port = {{ $notifyPort }} +mailer_host = {{ .notifier.smtp.host | quote }} +mailer_port = {{ .notifier.smtp.port }} +mailer_username = {{ .notifier.smtp.username | quote }} +mailer_password = {{ .notifier.smtp.password | quote }} +repo_sync_timezone = {{ .notifier.repo_sync_timezone | quote }} +broadcast_user_page_size = {{ .notifier.broadcast_user_page_size }} +broadcast_email_page_size = {{ .notifier.broadcast_email_page_size }} + +{{- $feishu := .notifier.feishu }} + +[feishu] +app_id = {{ $feishu.app_id | quote }} +app_secret = {{ $feishu.app_secret | quote }} +batch_send_message_cron_expression = {{ $feishu.batch_send_message_cron_expression | quote }} +max_request_content_size = {{ $feishu.max_request_content_size }} +max_delay_duration = {{ $feishu.max_delay_duration }} +chat_ids_cache_ttl = {{ $feishu.chat_ids_cache_ttl }} +{{- end }} + +{{- $prometheus := .server.prometheus }} + +[prometheus] +api_address = {{ printf "%s/api/v1/query" $prometheus.address | quote }} + +{{- $logcollector := .server.logcollector }} +[logcollector] +loki_url = {{ $logcollector.loki_url | quote }} + +{{- $agent := .server.agent }} + +[agent] +agent_hub_service_host = {{ $agent.langflow.token | quote }} +agent_hub_service_token = {{ $agent.langflow.host | quote }} +code_instance_quota_per_user = {{ $agent.code_instance_quota_per_user | default 5 }} +langflow_instance_quota_per_user = {{ $agent.instance_quota_per_user | default 5 }} + +{{- $csgbot := .server.csgbot }} + +[csgbot] +host = {{ $csgbot.host | quote }} +port = {{ $csgbot.port }} + +{{- $xnet := .server.xnet }} + +[xnet] +endpoint = {{ $xnet.endpoint | default (printf "%s/xnet" $csghub.url) | quote }} +api_key = {{ $xnet.api_key | default (deriveToken)| quote }} + +[aigateway] +port = {{ .aigateway.port }} +moderation_bypass_sensitive_check = {{ .aigateway.moderation_bypass_sensitive_check }} +advertise_addr = "{{ $csghub.host }}:8094" + +[storagegateway] +enable_presigned_url_proxy = {{ .server.enable_presigned_url_proxy }} diff --git a/ee/opt/csghub/etc/xnet/config.toml.sample b/ee/opt/csghub/embedded/etc/xnet/config.toml.sample similarity index 50% rename from ee/opt/csghub/etc/xnet/config.toml.sample rename to ee/opt/csghub/embedded/etc/xnet/config.toml.sample index a49f7258..a23f666f 100644 --- a/ee/opt/csghub/etc/xnet/config.toml.sample +++ b/ee/opt/csghub/embedded/etc/xnet/config.toml.sample @@ -1,10 +1,9 @@ -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -api_token = {{ tmpl.Exec "GenHubApiToken" . | quote }} -cas_url = {{ printf "%s/xnet" $csghub.external | quote }} +{{- $csghub := resolveUrl .csghub.external_url }} +api_token = {{ deriveToken | quote }} +cas_url = {{ printf "%s/xnet" $csghub.url | quote }} -{{ $xnet := (datasource "config").xnet -}} [xnet_server] -port = {{ $xnet.listen_port }} +port = {{ .xnet.listen_port }} # Only one of fdb and pg is active; If you compile using tags=fdb, no other configuration is needed. [fdb] @@ -19,24 +18,27 @@ tx_timeout_ms = 10000 max_retry_attempts = 5 retry_backoff_ms = 20 -{{ $postgresql := $xnet.postgresql }} -{{- $pgUser := $postgresql.user -}} -{{- $pgPassword := $postgresql.password | default (crypto.PBKDF2 $pgUser "opencsg" 2048 8) -}} +{{- $postgresql := .xnet.postgresql }} +{{- $pgUser := $postgresql.user }} +{{- $pgPassword := $postgresql.password | default (pbkdf2 $pgUser "opencsg" 2048 8) }} + [database] driver = "pg" -dsn = {{ printf "postgresql://%s:%s@%s:%d/%s?sslmode=disable" (conv.ToString $pgUser) (conv.ToString $pgPassword) (conv.ToString $postgresql.host) (conv.ToInt64 $postgresql.port) (conv.ToString $postgresql.name) | quote }} +dsn = {{ printf "postgresql://%s:%s@%s:%d/%s?sslmode=disable" (toString $pgUser) (toString $pgPassword) (toString $postgresql.host) (int64 $postgresql.port) (toString $postgresql.name) | quote }} timezone = {{ $postgresql.timezone | quote }} -{{ $s3 := $xnet.s3 -}} +{{- $s3 := .xnet.s3 }} + [xorb] meta_store = "s3" {{- $s3Endpoint := $s3.endpoint | default (printf "%s:9000" $csghub.host) }} xorb_s3_url = {{ printf "%s/%s" $s3Endpoint $s3.bucket | quote }} -signed_ul_expire_hours = {{ $xnet.s3.signed_ul_expire_hours | default 3 }} +signed_ul_expire_hours = {{ .xnet.s3.signed_ul_expire_hours | default 3 }} cleanup_timeout_seconds = 30 -{{ $accessKeyId := $s3.access_key -}} -{{- $secretKey := $s3.secret_key | default (crypto.PBKDF2 $accessKeyId "opencsg" 2048 8) -}} +{{- $accessKeyId := $s3.access_key }} +{{- $secretKey := $s3.secret_key | default (pbkdf2 $accessKeyId "opencsg" 2048 8) }} + [s3] access_key_id = {{ $accessKeyId | quote }} access_key_secret = {{ $secretKey | quote }} @@ -50,26 +52,25 @@ bucket_lookup = "path" {{- else }} bucket_lookup = "auto" {{- end }} -{{ $s3 := (datasource "config").server.s3 }} -lfs_bucket = {{ $s3.bucket | quote }} +lfs_bucket = {{ .server.s3.bucket | quote }} # Provider type: minio, aws, aliyun, isilon, rustfs, other etc. provider = "" [license] -public_key_file = "/opt/csghub/etc/server/enterprise/public_key_ee.pem" +public_key_file = "/opt/csghub/embedded/etc/server/enterprise/public_key_ee.pem" [jwt] -signing_key = {{ tmpl.Exec "GenSeed" . | crypto.SHA256 | quote }} +signing_key = {{ persistentSeed | sha256sum | quote }} valid_hour = 24 [instrumentation] otlp_endpoint = "{otlp_endpoint}" otlp_logging = false -{{ $nats := (datasource "config").nats }} -{{- $natsUser := $nats.auth.user -}} -{{- $natsPassword := $nats.auth.password | default (crypto.PBKDF2 $natsUser "opencsg" 2048 8) -}} -{{- $xnetNats := $xnet.nats -}} +{{- $natsUser := .nats.auth.user }} +{{- $natsPassword := .nats.auth.password | default (pbkdf2 $natsUser "opencsg" 2048 8) }} +{{- $xnetNats := .xnet.nats }} + [nats] -url = {{ $xnetNats.url | default (printf "nats://%s:%s@localhost:4222" (conv.ToString $natsUser) (conv.ToString $natsPassword)) | quote }} +url = {{ $xnetNats.url | default (printf "nats://%s:%s@localhost:4222" (toString $natsUser) (toString $natsPassword)) | quote }} msg_fetch_timeout_in_sec = {{ $xnetNats.msg_fetch_timeout_in_sec }} diff --git a/ee/opt/csghub/embedded/sv/agentic/templates/log/run.sample b/ee/opt/csghub/embedded/sv/agentic/templates/log/run.sample index 74422530..38ca7d69 100755 --- a/ee/opt/csghub/embedded/sv/agentic/templates/log/run.sample +++ b/ee/opt/csghub/embedded/sv/agentic/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/agentic" }} -{{- if has (datasource "config").agentic "log" }} -{{- if has (datasource "config").agentic.log "dir" }} -{{- if (datasource "config").agentic.log.dir }} -{{- $logDir = (datasource "config").agentic.log.dir }} -{{- end }} +{{- if has .agentic "log" }} +{{- if .agentic.log.dir }} +{{- $logDir = .agentic.log.dir }} {{- end }} {{- end }} diff --git a/ee/opt/csghub/embedded/sv/agentic/templates/run.sample b/ee/opt/csghub/embedded/sv/agentic/templates/run.sample index dfc88c31..05527378 100755 --- a/ee/opt/csghub/embedded/sv/agentic/templates/run.sample +++ b/ee/opt/csghub/embedded/sv/agentic/templates/run.sample @@ -3,18 +3,12 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # fail on errors set -e -{{- $agentic := (datasource "config").agentic }} cd /opt/csghub/embedded/sv/agentic -exec chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ +exec chpst -e /opt/csghub/service/web/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ /opt/csghub/embedded/bin/hypercorn starship.main:starlette \ - --bind {{ $agentic.listen }} \ - --log-level info \ No newline at end of file + --bind {{ .agentic.listen }} \ + --log-level info diff --git a/ee/opt/csghub/embedded/sv/billing/templates/log/run.sample b/ee/opt/csghub/embedded/sv/billing/templates/log/run.sample index 1b110a9c..63c2cead 100755 --- a/ee/opt/csghub/embedded/sv/billing/templates/log/run.sample +++ b/ee/opt/csghub/embedded/sv/billing/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/billing" }} -{{- if has (datasource "config").billing "log" }} -{{- if has (datasource "config").billing.log "dir" }} -{{- if (datasource "config").billing.log.dir }} -{{- $logDir = (datasource "config").billing.log.dir }} -{{- end }} +{{- if has .billing "log" }} +{{- if .billing.log.dir }} +{{- $logDir = .billing.log.dir }} {{- end }} {{- end }} diff --git a/ee/opt/csghub/embedded/sv/billing/templates/run.sample b/ee/opt/csghub/embedded/sv/billing/templates/run.sample index 04dc34b0..f4824988 100755 --- a/ee/opt/csghub/embedded/sv/billing/templates/run.sample +++ b/ee/opt/csghub/embedded/sv/billing/templates/run.sample @@ -3,16 +3,11 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # fail on errors set -e -{{- $dataDir := (datasource "config").billing.data | default "/var/opt/csghub/billing" }} +{{- $dataDir := .billing.data | default "/var/opt/csghub/billing" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/billing -config={{ $dataDir }}/config.yaml \ No newline at end of file +exec chpst -e /opt/csghub/service/web/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/billing -config=/opt/csghub/service/billing/etc/config.yaml diff --git a/ee/opt/csghub/embedded/sv/portal/templates/run.sample b/ee/opt/csghub/embedded/sv/portal/templates/run.sample index 33187109..bee92053 100755 --- a/ee/opt/csghub/embedded/sv/portal/templates/run.sample +++ b/ee/opt/csghub/embedded/sv/portal/templates/run.sample @@ -6,33 +6,51 @@ exec 2>&1 # Attempt to change ulimit before the set -e flag, ignore failures ulimit -n 15000 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # Exit if execute with any errors set -e -readonly PORTAL_BIN="/opt/csghub/embedded/bin/csghub-portal" -readonly ENV_DIR="/opt/csghub/etc/portal/env" +readonly PORTAL_BIN=/opt/csghub/embedded/bin/csghub-portal +readonly ENV_DIR=/opt/csghub/service/portal/env + +{{ if or .postgresql.enable .patroni.enable }} +# Wait for PostgreSQL +echo "Waiting for PostgreSQL to become ready..." +while ! /opt/csghub/embedded/sv/postgresql/bin/pg_isready -q; do sleep 2; done +echo "PostgreSQL is ready." +DB_USER=$(cat "$ENV_DIR/DB_USER") +DB_PASSWORD=$(cat "$ENV_DIR/DB_PASSWORD") +DB_NAME=$(cat "$ENV_DIR/DB_NAME") +# Create database and update permissions for portal user +/opt/csghub/bin/csghub-dbm -a create -d "$DB_NAME" -u "$DB_USER" -p "$DB_PASSWORD" +/opt/csghub/bin/csghub-dbm -a update -d "$DB_NAME" -N "$DB_USER" +{{- end }} + +{{ if .minio.enable }} +# Wait for MinIO and create buckets +echo "Waiting for MinIO to become ready..." +while ! /opt/csghub/service/minio/ping_minio; do sleep 2; done +echo "MinIO is ready." +# Create both public and private buckets for portal +/opt/csghub/service/minio/create_bucket {{ .portal.s3.bucket | default "csghub-portal-public" }} public +/opt/csghub/service/minio/create_bucket {{ .portal.additionalS3.bucket | default "csghub-portal" }} +{{- end }} echo "Initializing database migrations..." chpst -e "$ENV_DIR" -P \ - -u root:root \ + -u {{ .run_user }}:{{ .run_user }} \ "$PORTAL_BIN" migration init echo "Running database migrations..." chpst -e "$ENV_DIR" -P \ - -u root:root \ + -u {{ .run_user }}:{{ .run_user }} \ "$PORTAL_BIN" migration migrate echo "Seed database ..." chpst -e "$ENV_DIR" -P \ - -u root:root \ + -u {{ .run_user }}:{{ .run_user }} \ "$PORTAL_BIN" db seed echo "Starting portal..." exec chpst -e "$ENV_DIR" -P \ - -u root:root \ + -u {{ .run_user }}:{{ .run_user }} \ "$PORTAL_BIN" start server diff --git a/ee/opt/csghub/embedded/sv/producer/templates/log/run.sample b/ee/opt/csghub/embedded/sv/producer/templates/log/run.sample index 77ea3258..7c273d43 100755 --- a/ee/opt/csghub/embedded/sv/producer/templates/log/run.sample +++ b/ee/opt/csghub/embedded/sv/producer/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/producer" }} -{{- if has (datasource "config").producer "log" }} -{{- if has (datasource "config").producer.log "dir" }} -{{- if (datasource "config").producer.log.dir }} -{{- $logDir = (datasource "config").producer.log.dir }} -{{- end }} +{{- if has .producer "log" }} +{{- if .producer.log.dir }} +{{- $logDir = .producer.log.dir }} {{- end }} {{- end }} diff --git a/ee/opt/csghub/embedded/sv/producer/templates/run.sample b/ee/opt/csghub/embedded/sv/producer/templates/run.sample index d3ccd6b8..2312400e 100755 --- a/ee/opt/csghub/embedded/sv/producer/templates/run.sample +++ b/ee/opt/csghub/embedded/sv/producer/templates/run.sample @@ -3,15 +3,10 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # fail on errors set -e cd /opt/csghub/embedded/sv/web/project -exec chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/celery -A config beat --loglevel=INFO \ No newline at end of file +exec chpst -e /opt/csghub/service/web/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/celery -A config beat --loglevel=INFO diff --git a/ee/opt/csghub/embedded/sv/server/templates/pre-start.sample b/ee/opt/csghub/embedded/sv/server/templates/pre-start.sample deleted file mode 100755 index 2e7eeaa8..00000000 --- a/ee/opt/csghub/embedded/sv/server/templates/pre-start.sample +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# exit when error -set -e - -{{- $configDir := "/opt/csghub/etc/server" }} -{{- $envDir := printf "%s/env" $configDir }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -{{- file.Write (printf "%s/GIN_MODE" $envDir) "release" }} - -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $db := $server.postgresql }} -{{- $password := $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) }} -{{- file.Write (printf "%s/POSTGRES_USER" $envDir) $db.user }} -{{- file.Write (printf "%s/POSTGRES_PASSWORD" $envDir) $password }} -{{- file.Write (printf "%s/POSTGRES_DB" $envDir) $db.name }} -{{- file.Write (printf "%s/HOME" $envDir) "/etc/csghub" }} -{{- file.Write (printf "%s/DSN" $envDir) (printf "postgresql://%s:%s@%s:%d/%s" (conv.ToString $db.user) (conv.ToString $password) (conv.ToString $db.host) (conv.ToInt64 $db.port) (conv.ToString $db.name)) }} -{{- file.Write (printf "%s/STARHUB_SERVER_PUBLIC_ROOT_DOMAIN" $envDir) " " }} -{{- file.Write (printf "%s/DUCKDB_HOME" $envDir) "/opt/csghub/embedded/sv/server" }} -{{- file.Write (printf "%s/DUCKDB_EXTENSION_DIRECTORY" $envDir) "/opt/csghub/embedded/sv/server" }} -{{- file.Write (printf "%s/OPENCSG_LICENSE_PUBLIC_KEY_FILE" $envDir) "/opt/csghub/etc/server/enterprise/public_key_ee.pem" }} -{{- file.Write (printf "%s/OPENCSG_LICENSE_PRIVATE_KEY_FILE" $envDir) "/opt/csghub/etc/server/enterprise/private_key_ee.pem" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_KUBE_SCHEDULER" $envDir) "volcano" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_NODE_RESOURCE_NAME" $envDir) "volcano.sh/node-vgpu-register" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_POD_RESOURCE_NAME" $envDir) "volcano.sh/vgpu-ids-new" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_RESOURCE_REQ_KEY" $envDir) "volcano.sh/vgpu-number" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_MEMORY_REQ_KEY" $envDir) "volcano.sh/vgpu-memory" }} - -{{- $postgresql := (datasource "config").postgresql }} -{{- $patroni := (datasource "config").patroni }} -{{- if or $postgresql.enable $patroni.enable }} -# Create server database (current default) -/opt/csghub/bin/csghub-dbm -a create -d {{ $db.name }} -u {{ $db.user }} -p {{ $password }} -/opt/csghub/bin/csghub-dbm -a promote -u {{ $db.user }} -P SUPERUSER -# If server database exists reset all objects owner to current db user -/opt/csghub/bin/csghub-dbm -a update -d {{ $db.name }} -N {{ $db.user }} - -{{- end }} - -{{- $dataDir := (datasource "config").server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} - -# Define constants -readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" - -# Build command arguments -## Migration init -args=("migration" "init") -[[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") - -# Database init -cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" || true - -## Migration migrate -args=("migration" "migrate") -[[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") - -# Database setup -cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" || true - -# Load local sql scripts -cd /opt/csghub/service/server -[ -x "/opt/csghub/etc/csghub/scripts/pg_executor" ] && chpst -e {{ $envDir }} /opt/csghub/etc/csghub/scripts/pg_executor - -{{- $minio := (datasource "config").minio }} -{{- if $minio.enable }} -# Create bucket -[ -x "/opt/csghub/service/minio/create_bucket" ] && /opt/csghub/service/minio/create_bucket {{ $server.s3.bucket }} -{{- end }} - -find {{ $configDir }} -mindepth 1 -maxdepth 1 ! -name '*.sample' ! -name 'seed' ! -name 'env' ! -name 'scripts' -exec cp -r {} {{ $dataDir }} \; -echo "copy configuration files done." - -{{- $multi_sync := $server.multi_sync -}} -{{- if $multi_sync.enabled }} - -{{- $logDir := "/var/log/csghub/server" }} -{{- if has (datasource "config").server "log" }} -{{- if has (datasource "config").server.log "dir" }} -{{- if (datasource "config").server.log.dir }} -{{- $logDir = (datasource "config").server.log.dir }} -{{- end }} -{{- end }} -{{- end }} - -# Check and kill any existing multi-sync processes before starting a new one -if pgrep -f "$SERVER_BIN sync sync-as-client" >/dev/null; then - pkill -f "$SERVER_BIN sync sync-as-client" - # Wait a moment for the process to terminate - sleep 1 -fi - -# Start multi-sync -args=("sync" "sync-as-client") -[[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") - -# Initialize multi-source synchronization -(cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" > {{ $logDir }}/multi-sync.log 2>&1)& -{{- end }} diff --git a/ee/opt/csghub/embedded/sv/web/templates/log/run.sample b/ee/opt/csghub/embedded/sv/web/templates/log/run.sample index eb3b6f0a..d0ee2cf6 100755 --- a/ee/opt/csghub/embedded/sv/web/templates/log/run.sample +++ b/ee/opt/csghub/embedded/sv/web/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/web" }} -{{- if has (datasource "config").web "log" }} -{{- if has (datasource "config").web.log "dir" }} -{{- if (datasource "config").web.log.dir }} -{{- $logDir = (datasource "config").web.log.dir }} -{{- end }} +{{- if has .web "log" }} +{{- if .web.log.dir }} +{{- $logDir = .web.log.dir }} {{- end }} {{- end }} diff --git a/ee/opt/csghub/embedded/sv/web/templates/pre-start.sample b/ee/opt/csghub/embedded/sv/web/templates/pre-start.sample deleted file mode 100755 index d8bff0cb..00000000 --- a/ee/opt/csghub/embedded/sv/web/templates/pre-start.sample +++ /dev/null @@ -1,214 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $envDir := "/opt/csghub/etc/web/env" }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -{{- file.Write (printf "%s/PYTHONHOME" $envDir) "/opt/csghub/embedded/python" }} -{{- file.Write (printf "%s/LD_LIBRARY_PATH" $envDir) "/opt/csghub/embedded/python/lib:$LD_LIBRARY_PATH" }} -{{- file.Write (printf "%s/PYTHONPATH" $envDir) "/opt/csghub/embedded/python/lib/python3.11/site-packages" }} -{{- file.Write (printf "%s/PATH" $envDir) "/opt/csghub/embedded/python/bin:$PATH" }} - -{{- file.Write (printf "%s/DJANGO_DEBUG" $envDir) "false" }} -{{- file.Write (printf "%s/DJANGO_SETTINGS_MODULE" $envDir) "config.settings" }} -{{- file.Write (printf "%s/C_FORCE_ROOT" $envDir) "true" }} - -{{- $csgship := (datasource "config").csgship }} -{{- $web := (datasource "config").web }} -{{- $dataDir := $web.data | default "/var/opt/csghub/web" }} -{{- $db := $web.postgresql }} -{{- $password := $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) }} -{{- file.Write (printf "%s/DATABASE_TYPE" $envDir) "postgresql" }} -{{- file.Write (printf "%s/DATABASE_HOST" $envDir) $db.host }} -{{- file.Write (printf "%s/DATABASE_PORT" $envDir) $db.port }}#!/bin/bash - - # Redirect stderr -> stdout - exec 2>&1 - - # Exit if execute with any errors - set -e - - {{- $envDir := "/opt/csghub/etc/web/env" }} - - # Clear all environment variable files - # rm -rf {{ $envDir }}/* || true - - {{- file.Write (printf "%s/PYTHONHOME" $envDir) "/opt/csghub/embedded/python" }} - {{- file.Write (printf "%s/LD_LIBRARY_PATH" $envDir) "/opt/csghub/embedded/python/lib:$LD_LIBRARY_PATH" }} - {{- file.Write (printf "%s/PYTHONPATH" $envDir) "/opt/csghub/embedded/python/lib/python3.11/site-packages" }} - {{- file.Write (printf "%s/PATH" $envDir) "/opt/csghub/embedded/python/bin:$PATH" }} - - {{- file.Write (printf "%s/DJANGO_DEBUG" $envDir) "false" }} - {{- file.Write (printf "%s/DJANGO_SETTINGS_MODULE" $envDir) "config.settings" }} - {{- file.Write (printf "%s/C_FORCE_ROOT" $envDir) "true" }} - - {{- $csgship := (datasource "config").csgship }} - {{- $web := (datasource "config").web }} - {{- $dataDir := $web.data | default "/var/opt/csghub/web" }} - {{- $db := $web.postgresql }} - {{- $password := $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) }} - {{- file.Write (printf "%s/DATABASE_TYPE" $envDir) "postgresql" }} - {{- file.Write (printf "%s/DATABASE_HOST" $envDir) $db.host }} - {{- file.Write (printf "%s/DATABASE_PORT" $envDir) $db.port }} - {{- file.Write (printf "%s/DATABASE_USERNAME" $envDir) $db.user }} - {{- file.Write (printf "%s/DATABASE_PASSWORD" $envDir) $password }} - {{- file.Write (printf "%s/DATABASE_NAME" $envDir) $db.name }} - {{- file.Write (printf "%s/DSN" $envDir) (printf "postgresql://%s:%s@%s:%d/%s" (conv.ToString $db.user) (conv.ToString $password) (conv.ToString $db.host) (conv.ToInt64 $db.port) (conv.ToString $db.name)) }} - - {{- $postgresql := (datasource "config").postgresql }} - {{- $patroni := (datasource "config").patroni }} - {{- if or $postgresql.enable $patroni.enable }} - # Rename database csgship (If database csgship exists) - /opt/csghub/bin/csghub-dbm -a rename -d "starship_codegpt" -n {{ $db.name }} - /opt/csghub/bin/csghub-dbm -a rename -d "starship" -n {{ $db.name }} - # Create csgship database (current default) - /opt/csghub/bin/csghub-dbm -a create -d {{ $db.name }} -u {{ $db.user }} -p {{ $password }} - # If csgship database exists reset all objects owner to current db user - /opt/csghub/bin/csghub-dbm -a update -d {{ $db.name }} -N {{ $db.user }} - {{- end }} - - {{- $redis := $web.redis }} - {{- $redisUser := $redis.auth.user -}} - {{- $redisPassword := $redis.auth.password | default (crypto.PBKDF2 $redisUser "opencsg" 2048 8) -}} - {{- file.Write (printf "%s/REDIS_URL" $envDir) (printf ":@%s" $redis.host) }} - {{/*{{- file.Write (printf "%s/REDIS_URL" $envDir) (printf ":%s@%s" $redisPassword $redis.host) }} */}} - {{- file.Write (printf "%s/REDIS_PORT" $envDir) $redis.port }} - - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML }} - {{- file.Write (printf "%s/csgship_WEB_URL" $envDir) (tmpl.Exec "endpoint.csgship.frontend" .) }} - {{- file.Write (printf "%s/csgship_API_URL" $envDir) (tmpl.Exec "endpoint.csgship.api" .) }} - {{- file.Write (printf "%s/USE_OPENAI_ASSISTANTS_API" $envDir) "false" }} - - {{- file.Write (printf "%s/ACCOUNTING_OP_MODE" $envDir) "has_balance|consume" }} - {{- file.Write (printf "%s/ACCOUNTING_API_BASE" $envDir) $csghub.external }} - - {{- $apiToken := tmpl.Exec "GenHubApiToken" . }} - {{- file.Write (printf "%s/ACCOUNTING_API_KEY" $envDir) $apiToken }} - {{- file.Write (printf "%s/OPENCSG_HUB_BASE_URL" $envDir) $csghub.external }} - {{- file.Write (printf "%s/OPENCSG_HUB_ACCESS_TOKEN" $envDir) $apiToken }} - - {{- $openai := $web.openai }} - {{- file.Write (printf "%s/AZURE_MODEL" $envDir) $openai.model }} - {{- file.Write (printf "%s/AZURE_API_BASE" $envDir) $openai.api.base }} - {{- file.Write (printf "%s/AZURE_API_VERSION" $envDir) $openai.api.version }} - {{- file.Write (printf "%s/AZURE_API_KEY" $envDir) $openai.api.key }} - {{- file.Write (printf "%s/LANCEDB_URI" $envDir) "/opt/csghub/embedded/sv/web/data/lancedb" }} - - {{ $billing := (datasource "config").billing }} - {{ $agentic := (datasource "config").agentic }} - {{ $web := (datasource "config").web }} - {{- file.Write (printf "%s/csgship_BILLING_API_BASE" $envDir) (printf "http://%s" $billing.listen) }} - {{- file.Write (printf "%s/BILLING_SVC_API_BASE" $envDir) (printf "http://%s" $billing.listen) }} - {{- file.Write (printf "%s/AGENTIC_SVC_API_BASE" $envDir) (printf "http://%s" $agentic.listen) }} - {{- file.Write (printf "%s/WEB_SVC_API_BASE" $envDir) (printf "http://%s" $web.listen) }} - {{- file.Write (printf "%s/MEGALINTER_SVC_API_BASE" $envDir) $web.megalinter_server.endpoint }} - {{- file.Write (printf "%s/SYNC_PERMS_GIT_SERVER" $envDir) ($csgship.oauth.issuer | strings.TrimPrefix "http://" | strings.TrimPrefix "https://") }} - - {{ $nats := (datasource "config").nats }} - {{- $natsUser := $nats.auth.user -}} - {{- $natsPassword := $nats.auth.password | default (crypto.PBKDF2 $natsUser "opencsg" 2048 8) -}} - {{- $webNats := $web.nats -}} - {{- file.Write (printf "%s/NATS_URL" $envDir) ($webNats.url | default (printf "nats://%s:%s@localhost:4222" $natsUser $natsPassword)) }} - - {{- file.Write (printf "%s/GIT_PYTHON_GIT_EXECUTABLE" $envDir) "/opt/csghub/embedded/bin/gitaly-git-v2.47" }} - chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/sv/web/project/manage.py migrate - - chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/sv/web/project/manage.py createcachetable - - chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/sv/web/project/manage.py clearsessions - - # Load local sql scripts - cd /opt/csghub/service/web - [ -x "/opt/csghub/etc/csghub/scripts/pg_executor" ] && chpst -e {{ $envDir }} /opt/csghub/etc/csghub/scripts/pg_executor - - -{{- file.Write (printf "%s/DATABASE_USERNAME" $envDir) $db.user }} -{{- file.Write (printf "%s/DATABASE_PASSWORD" $envDir) $password }} -{{- file.Write (printf "%s/DATABASE_NAME" $envDir) $db.name }} -{{- file.Write (printf "%s/DSN" $envDir) (printf "postgresql://%s:%s@%s:%d/%s" (conv.ToString $db.user) (conv.ToString $password) (conv.ToString $db.host) (conv.ToInt64 $db.port) (conv.ToString $db.name)) }} - -{{- $postgresql := (datasource "config").postgresql }} -{{- $patroni := (datasource "config").patroni }} -{{- if or $postgresql.enable $patroni.enable }} -# Rename database csgship (If database csgship exists) -/opt/csghub/bin/csghub-dbm -a rename -d "starship_codegpt" -n {{ $db.name }} -/opt/csghub/bin/csghub-dbm -a rename -d "starship" -n {{ $db.name }} -# Create csgship database (current default) -/opt/csghub/bin/csghub-dbm -a create -d {{ $db.name }} -u {{ $db.user }} -p {{ $password }} -# If csgship database exists reset all objects owner to current db user -/opt/csghub/bin/csghub-dbm -a update -d {{ $db.name }} -N {{ $db.user }} -{{- end }} - -{{- $redis := $web.redis }} -{{- $redisUser := $redis.auth.user -}} -{{- $redisPassword := $redis.auth.password | default (crypto.PBKDF2 $redisUser "opencsg" 2048 8) -}} -{{- file.Write (printf "%s/REDIS_URL" $envDir) (printf ":@%s" $redis.host) }} -{{/*{{- file.Write (printf "%s/REDIS_URL" $envDir) (printf ":%s@%s" $redisPassword $redis.host) }} */}} -{{- file.Write (printf "%s/REDIS_PORT" $envDir) $redis.port }} - -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML }} -{{- file.Write (printf "%s/csgship_WEB_URL" $envDir) (tmpl.Exec "endpoint.csgship.frontend" .) }} -{{- file.Write (printf "%s/csgship_API_URL" $envDir) (tmpl.Exec "endpoint.csgship.api" .) }} -{{- file.Write (printf "%s/USE_OPENAI_ASSISTANTS_API" $envDir) "false" }} - -{{- file.Write (printf "%s/ACCOUNTING_OP_MODE" $envDir) "has_balance|consume" }} -{{- file.Write (printf "%s/ACCOUNTING_API_BASE" $envDir) $csghub.external }} - -{{- $apiToken := tmpl.Exec "GenHubApiToken" . }} -{{- file.Write (printf "%s/ACCOUNTING_API_KEY" $envDir) $apiToken }} -{{- file.Write (printf "%s/OPENCSG_HUB_BASE_URL" $envDir) $csghub.external }} -{{- file.Write (printf "%s/OPENCSG_HUB_ACCESS_TOKEN" $envDir) $apiToken }} - -{{- $openai := $web.openai }} -{{- file.Write (printf "%s/AZURE_MODEL" $envDir) $openai.model }} -{{- file.Write (printf "%s/AZURE_API_BASE" $envDir) $openai.api.base }} -{{- file.Write (printf "%s/AZURE_API_VERSION" $envDir) $openai.api.version }} -{{- file.Write (printf "%s/AZURE_API_KEY" $envDir) $openai.api.key }} -{{- file.Write (printf "%s/LANCEDB_URI" $envDir) "/opt/csghub/embedded/sv/web/data/lancedb" }} - -{{ $billing := (datasource "config").billing }} -{{ $agentic := (datasource "config").agentic }} -{{ $web := (datasource "config").web }} -{{- file.Write (printf "%s/csgship_BILLING_API_BASE" $envDir) (printf "http://%s" $billing.listen) }} -{{- file.Write (printf "%s/BILLING_SVC_API_BASE" $envDir) (printf "http://%s" $billing.listen) }} -{{- file.Write (printf "%s/AGENTIC_SVC_API_BASE" $envDir) (printf "http://%s" $agentic.listen) }} -{{- file.Write (printf "%s/WEB_SVC_API_BASE" $envDir) (printf "http://%s" $web.listen) }} -{{- file.Write (printf "%s/MEGALINTER_SVC_API_BASE" $envDir) $web.megalinter_server.endpoint }} -{{- file.Write (printf "%s/SYNC_PERMS_GIT_SERVER" $envDir) ($csgship.oauth.issuer | strings.TrimPrefix "http://" | strings.TrimPrefix "https://") }} - -{{ $nats := (datasource "config").nats }} -{{- $natsUser := $nats.auth.user -}} -{{- $natsPassword := $nats.auth.password | default (crypto.PBKDF2 $natsUser "opencsg" 2048 8) -}} -{{- $webNats := $web.nats -}} -{{- file.Write (printf "%s/NATS_URL" $envDir) ($webNats.url | default (printf "nats://%s:%s@localhost:4222" $natsUser $natsPassword)) }} - -{{- file.Write (printf "%s/GIT_PYTHON_GIT_EXECUTABLE" $envDir) "/opt/csghub/embedded/bin/gitaly-git-v2.47" }} -chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/sv/web/project/manage.py migrate - -chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/sv/web/project/manage.py createcachetable - -chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/sv/web/project/manage.py clearsessions - -# Load local sql scripts -cd /opt/csghub/service/web -[ -x "/opt/csghub/etc/csghub/scripts/pg_executor" ] && chpst -e {{ $envDir }} /opt/csghub/etc/csghub/scripts/pg_executor - diff --git a/ee/opt/csghub/embedded/sv/web/templates/run.sample b/ee/opt/csghub/embedded/sv/web/templates/run.sample index 27a59216..e04af3e5 100755 --- a/ee/opt/csghub/embedded/sv/web/templates/run.sample +++ b/ee/opt/csghub/embedded/sv/web/templates/run.sample @@ -3,16 +3,41 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # fail on errors set -e -{{- $dataDir := (datasource "config").web.data | default "/var/opt/csghub/web" }} +readonly ENV_DIR=/opt/csghub/service/web/env + +{{ if or .postgresql.enable .patroni.enable }} +# Wait for PostgreSQL +echo "Waiting for PostgreSQL to become ready..." +while ! /opt/csghub/embedded/sv/postgresql/bin/pg_isready -q; do sleep 2; done +echo "PostgreSQL is ready." +DB_USER=$(cat "$ENV_DIR/DATABASE_USERNAME") +DB_PASSWORD=$(cat "$ENV_DIR/DATABASE_PASSWORD") +DB_NAME=$(cat "$ENV_DIR/DATABASE_NAME") +# Create database and update permissions for web user +/opt/csghub/bin/csghub-dbm -a rename -d starship_codegpt -n "$DB_NAME" +/opt/csghub/bin/csghub-dbm -a rename -d starship -n "$DB_NAME" +/opt/csghub/bin/csghub-dbm -a create -d "$DB_NAME" -u "$DB_USER" -p "$DB_PASSWORD" +/opt/csghub/bin/csghub-dbm -a update -d "$DB_NAME" -N "$DB_USER" +{{- end }} + +# Django management commands +readonly PYTHON_BIN=/opt/csghub/embedded/bin/python3.11 +readonly MANAGE_PY=/opt/csghub/embedded/sv/web/project/manage.py +echo "Running django migrate..." +chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$PYTHON_BIN" "$MANAGE_PY" migrate +echo "Running django createcachetable..." +chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$PYTHON_BIN" "$MANAGE_PY" createcachetable +echo "Running django clearsessions..." +chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$PYTHON_BIN" "$MANAGE_PY" clearsessions + +# Execute SQL seed scripts (after Django migrations so tables exist) +cd /opt/csghub/service/web && chpst -e "$ENV_DIR" /opt/csghub/embedded/etc/csghub/scripts/pg_executor + +{{ $dataDir := .web.data | default "/var/opt/csghub/web" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/gunicorn -c /opt/csghub/embedded/sv/web/project/gunicorn_config.py \ No newline at end of file +exec chpst -e /opt/csghub/service/web/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/gunicorn -c /opt/csghub/embedded/sv/web/project/gunicorn_config.py diff --git a/ee/opt/csghub/embedded/sv/web/templates/scripts/06_socialaccount_socialapp_sites.sql.sample b/ee/opt/csghub/embedded/sv/web/templates/scripts/06_socialaccount_socialapp_sites.sql.sample index 12d4fa80..cc7cfa5b 100644 --- a/ee/opt/csghub/embedded/sv/web/templates/scripts/06_socialaccount_socialapp_sites.sql.sample +++ b/ee/opt/csghub/embedded/sv/web/templates/scripts/06_socialaccount_socialapp_sites.sql.sample @@ -11,6 +11,6 @@ pg_catalog.setval('public.socialaccount_socialapp_sites_id_seq', ( MAX(id) FROM public.socialaccount_socialapp_sites), TRUE); -{{- $csgshipApiEndpoint := tmpl.Exec "endpoint.csgship.api" . -}} -- Change django_site to the correct domain -UPDATE django_site SET domain = '{{ $csgshipApiEndpoint }}', name = '{{ $csgshipApiEndpoint }}' WHERE id = 1; \ No newline at end of file + {{- $csgshipAPI := resolveEndpoint $csghub.scheme $csghub.host .csgship.listen.api "127.0.0.1:8002" }} +UPDATE django_site SET domain = '{{ $csgshipAPI.url }}', name = '{{ $csgshipAPI.url }}' WHERE id = 1; diff --git a/ee/opt/csghub/embedded/sv/worker/templates/log/run.sample b/ee/opt/csghub/embedded/sv/worker/templates/log/run.sample index 9199a5b2..f4d0c429 100755 --- a/ee/opt/csghub/embedded/sv/worker/templates/log/run.sample +++ b/ee/opt/csghub/embedded/sv/worker/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/worker" }} -{{- if has (datasource "config").worker "log" }} -{{- if has (datasource "config").worker.log "dir" }} -{{- if (datasource "config").worker.log.dir }} -{{- $logDir = (datasource "config").worker.log.dir }} -{{- end }} +{{- if has .worker "log" }} +{{- if .worker.log.dir }} +{{- $logDir = .worker.log.dir }} {{- end }} {{- end }} diff --git a/ee/opt/csghub/embedded/sv/worker/templates/run.sample b/ee/opt/csghub/embedded/sv/worker/templates/run.sample index 228f1181..40b99e22 100755 --- a/ee/opt/csghub/embedded/sv/worker/templates/run.sample +++ b/ee/opt/csghub/embedded/sv/worker/templates/run.sample @@ -3,15 +3,10 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # fail on errors set -e cd /opt/csghub/embedded/sv/web/project -exec chpst -e /opt/csghub/etc/web/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/celery -A config worker --loglevel=INFO -Q high_priority_queue,default -c 2 -n worker%%h \ No newline at end of file +exec chpst -e /opt/csghub/service/web/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/celery -A config worker --loglevel=INFO -Q high_priority_queue,default -c 2 -n worker%%h diff --git a/ee/opt/csghub/embedded/sv/xnet/templates/log/run.sample b/ee/opt/csghub/embedded/sv/xnet/templates/log/run.sample index e051445f..eda0f891 100755 --- a/ee/opt/csghub/embedded/sv/xnet/templates/log/run.sample +++ b/ee/opt/csghub/embedded/sv/xnet/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/xnet" }} -{{- if has (datasource "config").xnet "log" }} -{{- if has (datasource "config").xnet.log "dir" }} -{{- if (datasource "config").xnet.log.dir }} -{{- $logDir = (datasource "config").xnet.log.dir }} -{{- end }} +{{- if has .xnet "log" }} +{{- if .xnet.log.dir }} +{{- $logDir = .xnet.log.dir }} {{- end }} {{- end }} diff --git a/ee/opt/csghub/embedded/sv/xnet/templates/pre-start.sample b/ee/opt/csghub/embedded/sv/xnet/templates/pre-start.sample deleted file mode 100755 index 09b79b8c..00000000 --- a/ee/opt/csghub/embedded/sv/xnet/templates/pre-start.sample +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# exit when error -set -e - -{{- $configDir := "/opt/csghub/etc/xnet" }} -{{- $envDir := printf "%s/env" $configDir }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -{{- $xnet := (datasource "config").xnet }} -{{- $dataDir := $xnet.data | default "/var/opt/csghub/xnet" }} -{{- $db := $xnet.postgresql }} -{{- $password := $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) }} - -{{- $postgresql := (datasource "config").postgresql }} -{{- $patroni := (datasource "config").patroni }} -{{- if or $postgresql.enable $patroni.enable }} -# Create xnet database (current default) -/opt/csghub/bin/csghub-dbm -a create -d {{ $db.name }} -u {{ $db.user }} -p {{ $password }} -{{- end }} - -{{- $minio := (datasource "config").minio }} -{{- if $minio.enable }} -# Create bucket -[ -x "/opt/csghub/service/minio/create_bucket" ] && /opt/csghub/service/minio/create_bucket {{ $xnet.s3.bucket }} -{{- end }} diff --git a/ee/opt/csghub/embedded/sv/xnet/templates/run.sample b/ee/opt/csghub/embedded/sv/xnet/templates/run.sample index 35ec4a00..8fe52cde 100755 --- a/ee/opt/csghub/embedded/sv/xnet/templates/run.sample +++ b/ee/opt/csghub/embedded/sv/xnet/templates/run.sample @@ -6,23 +6,40 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi +readonly ENV_DIR=/opt/csghub/service/xnet/env -{{- $xnet := (datasource "config").xnet }} -{{- $dataDir := $xnet.data | default "/var/opt/csghub/xnet" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{ if or .postgresql.enable .patroni.enable }} +# Wait for PostgreSQL +echo "Waiting for PostgreSQL to become ready..." +while ! /opt/csghub/embedded/sv/postgresql/bin/pg_isready -q; do sleep 2; done +echo "PostgreSQL is ready." +DB_USER=$(cat "$ENV_DIR/DB_USER") +DB_PASSWORD=$(cat "$ENV_DIR/DB_PASSWORD") +DB_NAME=$(cat "$ENV_DIR/DB_NAME") +# Create database and update permissions for xnet user +/opt/csghub/bin/csghub-dbm -a create -d "$DB_NAME" -u "$DB_USER" -p "$DB_PASSWORD" +{{- end }} + +{{ if .minio.enable }} +# Wait for MinIO and create bucket +echo "Waiting for MinIO to become ready..." +while ! /opt/csghub/service/minio/ping_minio; do sleep 2; done +echo "MinIO is ready." +# Create bucket for xnet +/opt/csghub/service/minio/create_bucket {{ .xnet.s3.bucket | default "csghub-xnet" }} +{{- end }} + +{{- $dataDir := .xnet.data | default "/var/opt/csghub/xnet" }} +{{- $configDir := "/opt/csghub/service/xnet/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-xnet" -readonly ENV_DIR="/opt/csghub/etc/xnet/env" # Build command arguments -args=("start" "server" "-l" {{ $xnet.log.level | quote }}) +args=("start" "server" "-l" {{ .xnet.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting xnet..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/ee/opt/csghub/embedded/templates/csgship.tpl b/ee/opt/csghub/embedded/templates/csgship.tpl deleted file mode 100644 index 1769546b..00000000 --- a/ee/opt/csghub/embedded/templates/csgship.tpl +++ /dev/null @@ -1,51 +0,0 @@ -{{- define "config.csgship.frontend" -}} - {{- $csgship := (datasource "config").csgship -}} - {{- $csgshipFrontendListen := $csgship.listen.frontend -}} - {{- $csgshipFrontendParts := $csgshipFrontendListen | strings.Split ":" -}} - {{- $csgshipFrontendHost := index $csgshipFrontendParts 0 -}} - - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} - {{- if eq $csgshipFrontendHost "127.0.0.1" -}} - {{- $csgshipFrontendHost = $csghub.host -}} - {{- end -}} - - {{- $csgshipFrontendPort := "8001" -}} - {{- if eq (len $csgshipFrontendParts) 2 -}} - {{- $csgshipFrontendPort = index $csgshipFrontendParts 1 -}} - {{- end }} - - {{- $result := coll.Dict "host" $csgshipFrontendHost "port" $csgshipFrontendPort -}} - {{- $result | data.ToYAML -}} -{{- end }} - -{{- define "endpoint.csgship.frontend" -}} - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} - {{- $csgshipFrontend := tmpl.Exec "config.csgship.frontend" . | data.YAML -}} - {{- printf "%s://%s:%v" $csghub.scheme $csghub.host $csgshipFrontend.port -}} -{{- end -}} - -{{- define "config.csgship.api" -}} - {{- $csgship := (datasource "config").csgship -}} - {{- $csgshipApiListen := $csgship.listen.api -}} - {{- $csgshipApiParts := $csgshipApiListen | strings.Split ":" -}} - {{- $csgshipApiHost := index $csgshipApiParts 0 -}} - - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} - {{- if eq $csgshipApiHost "127.0.0.1" -}} - {{- $csgshipApiHost = $csghub.host -}} - {{- end -}} - - {{- $csgshipApiPort := "8002" -}} - {{- if eq (len $csgshipApiParts) 2 -}} - {{- $csgshipApiPort = index $csgshipApiParts 1 -}} - {{- end }} - - {{- $result := coll.Dict "host" $csgshipApiHost "port" $csgshipApiPort -}} - {{- $result | data.ToYAML -}} -{{- end -}} - -{{- define "endpoint.csgship.api" -}} - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} - {{- $csgshipApi := tmpl.Exec "config.csgship.api" . | data.YAML -}} - {{- printf "%s://%s:%v" $csghub.scheme $csghub.host $csgshipApi.port -}} -{{- end -}} diff --git a/ee/opt/csghub/etc/billing/config.yaml.sample b/ee/opt/csghub/etc/billing/config.yaml.sample deleted file mode 100644 index a83cedaf..00000000 --- a/ee/opt/csghub/etc/billing/config.yaml.sample +++ /dev/null @@ -1,18 +0,0 @@ -{{ $nats := (datasource "config").nats }} -{{ $billing := (datasource "config").billing }} -{{ $web := (datasource "config").web }} -{{- $natsUser := $nats.auth.user -}} -{{- $natsPassword := $nats.auth.password | default (crypto.PBKDF2 $natsUser "opencsg" 2048 8) -}} -{{- $webNats := $web.nats -}} -nats: {{ $webNats.url | default (printf "nats://%s:%s@localhost:4222" $natsUser $natsPassword) }} -pub: - addr: {{ $billing.listen }} - key: '' -sub: - streams: - - name: accountingNotifyStream - consumers: - - durable_name: starship-billing-durable-consumer - webhook: - url: http://127.0.0.1:8001/api/v1/platforms/billing-events/ - key: 'key' \ No newline at end of file diff --git a/ee/opt/csghub/etc/nginx/html/config.js.sample b/ee/opt/csghub/etc/nginx/html/config.js.sample deleted file mode 100644 index c5bd985e..00000000 --- a/ee/opt/csghub/etc/nginx/html/config.js.sample +++ /dev/null @@ -1,7 +0,0 @@ -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -{{- $casdoor := tmpl.Exec "config.casdoor.conn" . | data.YAML }} -window.envConfig = { - VITE_APP_BASE_WEB: {{ tmpl.Exec "endpoint.csgship.frontend" . | squote }}, - VITE_APP_BASE_API: {{ tmpl.Exec "endpoint.csgship.api" . | squote }}, - VITE_APP_LOGIN_URL: {{ printf "%s://%s:%s/login/oauth/authorize?client_id=%s&response_type=code&redirect_uri=%s/api/v1/account/casdoor/login/callback&scope=profile&state=casdoor" $csghub.scheme $csghub.host $casdoor.port (tmpl.Exec "GenClientId" "CSGShip") $csghub.external | squote }} -}; \ No newline at end of file diff --git a/ee/opt/csghub/etc/server/config.toml.sample b/ee/opt/csghub/etc/server/config.toml.sample deleted file mode 100644 index 8a38b17e..00000000 --- a/ee/opt/csghub/etc/server/config.toml.sample +++ /dev/null @@ -1,318 +0,0 @@ -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -{{- $server := (datasource "config").server -}} -{{- $runner := (datasource "config").runner -}} -{{- $praefect := (datasource "config").praefect -}} -{{- $gitlabShell := (datasource "config").gitlab_shell -}} - -saas = {{ $server.saas }} -oversea = false -instance_id = "" -enable_swagger = {{ $server.enable_swagger }} -enable_https = {{ $server.enable_https }} -api_token = {{ tmpl.Exec "GenHubApiToken" . | quote }} -docs_host = {{ printf "%s://%s:6636" $csghub.scheme $csghub.host | quote }} -server_failure_redirect_url = {{ printf "%s/errors/server-error" $csghub.external | quote }} - -[api_server] -port = {{ $server.listen_port }} -public_domain = {{ $csghub.external | quote }} -{{- $shell_port := $gitlabShell.ssh_port }} -{{- if eq $shell_port 22 }} -ssh_domain = {{ printf "git@%s:%d" (conv.ToString $csghub.host) (conv.ToInt64 $shell_port) | quote }} -{{- else }} -ssh_domain = {{ printf "ssh://git@%s:%d" (conv.ToString $csghub.host) (conv.ToInt64 $shell_port) | quote }} -{{- end }} - -{{ $mirrorRepo := (datasource "config").mirror_repo }} -{{- $mirrorLfs := (datasource "config").mirror_lfs -}} -[mirror] -url = {{ printf "http://localhost:%d" $mirrorRepo.listen_port | quote }} -token = "" -port = {{ $mirrorRepo.listen_port }} -{{- if $server.saas }} -remote = true -{{- else }} -remote = false -{{- end }} -session_secret_key = {{ tmpl.Exec "GenSeed" . | crypto.SHA1 | quote }} -worker_number = {{ $mirrorRepo.worker_number }} -lfs_concurrency = {{ $mirrorLfs.lfs_concurrency }} -part_size = {{ $mirrorLfs.part_size }} - -{{ $postgresql := $server.postgresql }} -{{- $pgUser := $postgresql.user -}} -{{- $pgPassword := $postgresql.password | default (crypto.PBKDF2 $pgUser "opencsg" 2048 8) -}} -[database] -driver = "pg" -dsn = {{ printf "postgresql://%s:%s@%s:%d/%s?sslmode=disable" (conv.ToString $pgUser) (conv.ToString $pgPassword) (conv.ToString $postgresql.host) (conv.ToInt64 $postgresql.port) (conv.ToString $postgresql.name) | quote }} -timezone = {{ $postgresql.timezone | quote }} - -{{ $redis := $server.redis }} -{{- $redisUser := $redis.auth.user -}} -{{- $redisPassword := $redis.auth.password | default (crypto.PBKDF2 $redisUser "opencsg" 2048 8) -}} -[redis] -endpoint = {{ $redis.endpoint | quote }} -max_retries = {{ $redis.max_retries }} -min_idle_connections = {{ $redis.min_idle_connections }} -user = {{ $redisUser | quote }} -password = {{ $redisPassword | quote }} -sentinel_mode = {{ $redis.sentinel.mode }} -sentinel_endpoint = {{ $redis.sentinel.endpoint | quote }} -sentinel_master = {{ $redis.sentinel.master | quote }} - -[git_server] -type = "gitaly" - -{{ $gitaly := $server.gitaly }} -{{- $gitalyToken := $gitaly.token | default (crypto.PBKDF2 "gitaly" "opencsg" 2048 8) -}} -[gitaly_server] -address = {{ $gitaly.address | quote }} -storge = {{ $gitaly.storage | quote }} -token = {{ $gitalyToken | quote }} -jwt_secret ={{ $gitaly.secret | default (tmpl.Exec "GenSeed" . | crypto.SHA256) | quote }} - -[frontend] -url = {{ $csghub.external | quote }} - -{{ $s3 := $server.s3 }} -{{- $accessKeyId := $s3.access_key -}} -{{- $secretKey := $s3.secret_key | default (crypto.PBKDF2 $accessKeyId "opencsg" 2048 8) -}} -[s3] -access_key_id = {{ $accessKeyId | quote }} -access_key_secret = {{ $secretKey | quote }} -region = {{ $s3.region | quote }} -endpoint = {{ $s3.endpoint | default (printf "%s:9000" $csghub.host) | quote }} -internal_endpoint = {{ $s3.internal_endpoint | quote }} -bucket = {{ $s3.bucket | quote }} -enable_ssl = {{ $s3.secure }} -{{- if $s3.path_style }} -bucket_lookup = "path" -{{- else }} -bucket_lookup = "auto" -{{- end }} - -[jwt] -signing_key = {{ $gitaly.secret | default (tmpl.Exec "GenSeed" . | crypto.SHA256) | quote }} -valid_hour = 24 - -[space] -internal_root_domain = "spaces.app.internal:8083" -{{- if $runner.use_public_domain }} -public_root_domain = {{ printf "%s:%v" $csghub.host $csghub.port | quote }} -{{- else }} -public_root_domain = "" -{{- end }} -session_secret_key = {{ tmpl.Exec "GenSeed" . | crypto.SHA1 | quote }} -pypi_index_url = {{ $runner.pip_index_url | quote }} - -{{ $model := $runner.model -}} -[model] -deploy_timeout_in_min = {{ $model.deploy_timeout }} -download_endpoint = {{ $csghub.external | quote }} -docker_reg_base = {{ $model.registry | quote }} -nim_docker_secret_name = "ngc-secret" -nim_ngc_secret_name = "nvidia-nim-secrets" - -[event] -sync_interval = {{ $mirrorRepo.sync_interval }} - -{{ $casdoor := (datasource "config").casdoor -}} -{{- $serverCasdoor := $server.casdoor -}} -[casdoor] -client_id = {{ tmpl.Exec "GenClientId" "CSGHub" | quote }} -client_secret = {{ tmpl.Exec "GenClientSecret" "CSGHub" | quote }} -endpoint = {{ $serverCasdoor.endpoint | default (tmpl.Exec "endpoint.casdoor" .) | quote }} -certificate = {{ $serverCasdoor.certificate | quote }} -organization_name = {{ $serverCasdoor.organization_name | quote }} -application_name = {{ $serverCasdoor.application_name | quote }} - -{{ $nats := (datasource "config").nats }} -{{- $natsUser := $nats.auth.user -}} -{{- $natsPassword := $nats.auth.password | default (crypto.PBKDF2 $natsUser "opencsg" 2048 8) -}} -{{- $serverNats := $server.nats -}} -[nats] -url = {{ $serverNats.url | default (printf "nats://%s:%s@localhost:4222" (conv.ToString $natsUser) (conv.ToString $natsPassword)) | quote }} -msg_fetch_timeout_in_sec = {{ $serverNats.msg_fetch_timeout_in_sec }} - -{{ $accounting := (datasource "config").accounting }} -{{- $actParts := $accounting.listen | strings.Split ":" -}} -{{- $actHost := index $actParts 0 -}} -{{- $actPort := "" -}} -{{- if gt (len $actParts) 1 -}} - {{- $actPort = index $actParts 1 -}} -{{- else -}} - {{- $actPort = "8086" -}} -{{- end -}} -[accounting] -host = {{ printf "http://%s" $actHost | quote }} -port = {{ $actPort }} - -{{ $user := (datasource "config").user }} -{{- $userParts := $user.listen | strings.Split ":" -}} -{{- $userHost := index $userParts 0 -}} -{{- $userPort := "" -}} -{{- if gt (len $userParts) 1 -}} - {{- $userPort = index $userParts 1 -}} -{{- else -}} - {{- $actPort = "8088" -}} -{{- end -}} -[user] -host = {{ printf "http://%s" $userHost | quote }} -port = {{ $userPort }} -signin_success_redirect_url = {{ printf "%s/server/callback" $csghub.external | quote }} - -{{ $multi_sync := $server.multi_sync -}} -[multi_sync] -enabled = {{ $multi_sync.enabled }} -saas_api_domain = "https://hub.opencsg.com" -saas_sync_domain = "https://sync.opencsg.com" - -{{ $telemetry := $server.telemetry -}} -[telemetry] -enable = {{ $telemetry.enabled | default true }} -report_url = "http://hub.opencsg.com/api/v1/telemetry" - -[auto_clean] -instance = false - -{{ $dataset := $server.dataset -}} -[dataset] -prompt_max_jsonl_file_size = {{ $dataset.prompt_max_jsonl_file_size }} - -{{ $dataflow := $server.dataflow -}} -{{- $dfUrl := conv.URL $dataflow.address -}} -{{- $dfUrlParts := $dfUrl.Host | strings.Split ":" -}} -{{- $dfHost := index $dfUrlParts 0 -}} -{{- $dfPort := "" -}} -{{- if gt (len $dfUrlParts) 1 -}} - {{- $dfPort = index $dfUrlParts 1 -}} -{{- else -}} - {{- if eq $dfUrl.Scheme "https" -}} - {{- $dfPort = "443" -}} - {{- else -}} - {{- $dfPort = "80" -}} - {{- end -}} -{{- end -}} -[dataflow] -host = {{ printf "%s://%s" $dfUrl.Scheme $dfHost | quote }} -port = {{ $dfPort }} - -{{ $moderation := (datasource "config").moderation -}} -{{- if $moderation.enable -}} -{{- $moderationAddress := $moderation.address -}} -{{- $moderationUrl := conv.URL $moderationAddress -}} -{{- $moderationParts := $moderationUrl.Host | strings.Split ":" -}} -{{- $moderationHost := index $moderationParts 0 | default "127.0.0.1" -}} -{{- $moderationPort := "" -}} -{{- if gt (len $moderationParts) 1 }} - {{- $moderationPort = index $moderationParts 1 -}} -{{- else -}} - {{- $moderationPort = "8089" -}} -{{- end -}} -[moderation] -host = {{ printf "%s://%s" $moderationUrl.Scheme $moderationHost | quote }} -port = {{ $moderationPort }} -encoded_sensitive_words = {{ $moderation.encoded_sensitive_words | quote }} - -{{ $sensitive := $moderation.sensitive_check -}} -[sensitive_check] -enable = true -access_key_id = {{ $sensitive.access_key_id | quote }} -access_key_secret = {{ $sensitive.access_key_secret | quote }} -region = {{ $sensitive.region | quote }} -endpoint = {{ $sensitive.endpoint | quote }} -enable_ssl = {{ $sensitive.enable_ssl }} -{{- end -}} - -{{ $workflow := $server.workflow -}} -[workflow] -endpoint = {{ $workflow.endpoint | quote }} - -[cron_job] -sync_as_client_cron_expression = {{ $multi_sync.sync_as_client_cron_expression | quote }} -calc_recom_score_cron_expression = {{ $multi_sync.calc_recom_score_cron_expression | quote }} - -{{ $proxy := $server.proxy -}} -[proxy] -hosts = [{{- range $k, $v := $proxy.hosts }}{{if $k}}, {{end}}{{$v | quote }}{{- end }}] - -{{ $instrumentation := $server.instrumentation -}} -[instrumentation] -otlp_logging = {{ $instrumentation.otlp_logging }} -otlp_endpoint = {{ $instrumentation.otlp_endpoint | quote }} - -{{ $git := $server.git -}} -[git] -operation_timeout = {{ $git.operation_timeout }} -{{- if $praefect.enable }} -check_file_size_enabled = false -{{- else }} -check_file_size_enabled = {{ $git.check_file_size_enabled }} -{{- end }} -max_un_lfs_file_size = {{ $git.max_un_lfs_file_size }} -skip_lfs_file_validation = {{ $git.skip_lfs_file_validation }} - - -{{ $notifier := (datasource "config").notifier -}} -{{- if $notifier.enable }} -{{- $notifyListen := $notifier.listen -}} -{{- $notifyParts := $notifyListen | strings.Split ":" -}} -{{- $notifyHost := index $notifyParts 0 | default "127.0.0.1" -}} -{{- $notifyPort := "8095" -}} -{{- if eq (len $notifyParts) 2 -}} - {{- $notifyPort = index $notifyParts 1 -}} -{{- end -}} -[notification] -host = {{ printf "http://%s" $notifyHost | quote }} -port = {{ $notifyPort }} -mailer_host = {{ $notifier.smtp.host | quote }} -mailer_port = {{ $notifier.smtp.port }} -mailer_username = {{ $notifier.smtp.username | quote }} -mailer_password = {{ $notifier.smtp.password | quote }} -repo_sync_timezone = {{ $notifier.repo_sync_timezone | quote }} -broadcast_user_page_size = {{ $notifier.broadcast_user_page_size }} -broadcast_email_page_size = {{ $notifier.broadcast_email_page_size }} - -{{ $feishu := $notifier.feishu -}} -[feishu] -app_id = {{ $feishu.app_id | quote }} -app_secret = {{ $feishu.app_secret | quote }} -batch_send_message_cron_expression = {{ $feishu.batch_send_message_cron_expression | quote }} -max_request_content_size = {{ $feishu.max_request_content_size }} -max_delay_duration = {{ $feishu.max_delay_duration }} -chat_ids_cache_ttl = {{ $feishu.chat_ids_cache_ttl }} -{{- end }} - -{{ $prometheus := $server.prometheus -}} -[prometheus] -api_address = {{ printf "%s/api/v1/query" $prometheus.address | quote }} - -{{ $logcollector := $server.logcollector -}} -[logcollector] -loki_url = {{ $logcollector.loki_url | quote }} - -{{ $agent := $server.agent -}} -[agent] -agent_hub_service_host = {{ $agent.langflow.token | quote }} -agent_hub_service_token = {{ $agent.langflow.host | quote }} -code_instance_quota_per_user = {{ $agent.code_instance_quota_per_user | default 5 }} -langflow_instance_quota_per_user = {{ $agent.instance_quota_per_user | default 5 }} - -{{ $csgbot := $server.csgbot -}} -[csgbot] -host = {{ $csgbot.host | quote }} -port = {{ $csgbot.port }} - -{{ $xnet := $server.xnet -}} -[xnet] -endpoint = {{ $xnet.endpoint | default (printf "%s/xnet" $csghub.external) | quote }} -api_key = {{ $xnet.api_key | default (tmpl.Exec "GenHubApiToken" .)| quote }} - -{{ $aigateway := (datasource "config").aigateway -}} -[aigateway] -port = {{ $aigateway.port }} -moderation_bypass_sensitive_check = {{ $aigateway.moderation_bypass_sensitive_check }} -advertise_addr = "{{ $csghub.host }}:8094" - -[storagegateway] -enable_presigned_url_proxy = {{ $server.enable_presigned_url_proxy }} \ No newline at end of file diff --git a/opt/csghub/etc/casdoor/app.conf.sample b/opt/csghub/embedded/etc/casdoor/app.conf.sample similarity index 65% rename from opt/csghub/etc/casdoor/app.conf.sample rename to opt/csghub/embedded/etc/casdoor/app.conf.sample index b2328adc..95ffff3f 100644 --- a/opt/csghub/etc/casdoor/app.conf.sample +++ b/opt/csghub/embedded/etc/casdoor/app.conf.sample @@ -1,10 +1,11 @@ -{{- $casdoor := (datasource "config").casdoor -}} +{{- $csghub := resolveUrl .csghub.external_url }} +{{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} appname = casdoor httpport = 8087 runmode = prod copyrequestbody = true driverName = postgres -dataSourceName = {{ tmpl.Exec "config.casdoor.db.dsn" . | quote }} +dataSourceName = {{ (buildDBConfig .casdoor.postgresql).dsn | quote }} dbName = tableNamePrefix = showSql = false @@ -16,7 +17,7 @@ socks5Proxy = "127.0.0.1:10808" verificationCodeTimeout = 10 initScore = 0 logPostOnly = true -origin = {{ tmpl.Exec "endpoint.casdoor" . }} +origin = {{ $casdoorEp.url }} originFrontend = staticBaseUrl = "https://cdn.casbin.org" isDemoMode = false @@ -27,7 +28,7 @@ radiusServerPort = 1812 radiusSecret = "secret" quota = {"organization": -1, "user": -1, "application": -1, "provider": -1} logConfig = {"filename": "logs/casdoor.log", "maxdays":99999, "perm":"0777"} -initDataFile = "{{ $casdoor.data }}/init_data.json" +initDataFile = "/opt/csghub/service/casdoor/etc/init_data.json" initDataNewOnly = true -frontendBaseDir = "/opt/csghub/etc/casdoor" +frontendBaseDir = "/opt/csghub/embedded/etc/casdoor" inactiveTimeoutMinutes = 30 diff --git a/opt/csghub/etc/casdoor/files/avatar/OpenCSG/background.png b/opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/background.png similarity index 100% rename from opt/csghub/etc/casdoor/files/avatar/OpenCSG/background.png rename to opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/background.png diff --git a/opt/csghub/etc/casdoor/files/avatar/OpenCSG/favicon.png b/opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/favicon.png similarity index 100% rename from opt/csghub/etc/casdoor/files/avatar/OpenCSG/favicon.png rename to opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/favicon.png diff --git a/opt/csghub/etc/casdoor/files/avatar/OpenCSG/gitlab_favicon.png b/opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/gitlab_favicon.png similarity index 100% rename from opt/csghub/etc/casdoor/files/avatar/OpenCSG/gitlab_favicon.png rename to opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/gitlab_favicon.png diff --git a/opt/csghub/etc/casdoor/files/avatar/OpenCSG/logo.png b/opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/logo.png similarity index 100% rename from opt/csghub/etc/casdoor/files/avatar/OpenCSG/logo.png rename to opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/logo.png diff --git a/opt/csghub/etc/casdoor/files/avatar/OpenCSG/robot.png b/opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/robot.png similarity index 100% rename from opt/csghub/etc/casdoor/files/avatar/OpenCSG/robot.png rename to opt/csghub/embedded/etc/casdoor/files/avatar/OpenCSG/robot.png diff --git a/opt/csghub/etc/casdoor/init_data.json.sample b/opt/csghub/embedded/etc/casdoor/init_data.json.sample similarity index 87% rename from opt/csghub/etc/casdoor/init_data.json.sample rename to opt/csghub/embedded/etc/casdoor/init_data.json.sample index a4bdc444..af6f37f9 100644 --- a/opt/csghub/etc/casdoor/init_data.json.sample +++ b/opt/csghub/embedded/etc/casdoor/init_data.json.sample @@ -1,15 +1,14 @@ -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -{{- $csgship := (datasource "config").csgship -}} -{{- $casdoorEndpoint := tmpl.Exec "endpoint.casdoor" . -}} +{{- $csghub := resolveUrl .csghub.external_url }} +{{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} { "organizations": [ { "owner": "admin", "name": "OpenCSG", "displayName": "OpenCSG", - "websiteUrl": "{{ $csghub.external }}", - "favicon": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/favicon.png", - "logo": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/logo.png", + "websiteUrl": "{{ $csghub.url }}", + "favicon": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/favicon.png", + "logo": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/logo.png", "passwordType": "bcrypt", "passwordSalt": "OpenCSG", "passwordOptions": [ @@ -21,7 +20,7 @@ "CN", "US" ], - "defaultAvatar": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/robot.png", + "defaultAvatar": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/robot.png", "defaultApplication": "CSGHub", "tags": [], "languages": [ @@ -150,8 +149,8 @@ "owner": "admin", "name": "CSGHub", "displayName": "CSGHub", - "logo": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/logo.png", - "homepageUrl": "{{ $csghub.external }}", + "logo": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/logo.png", + "homepageUrl": "{{ $csghub.url }}", "organization": "OpenCSG", "cert": "csghub-cert", "enablePassword": true, @@ -166,8 +165,8 @@ "enableLinkWithEmail": true, "orgChoiceMode": "Select", "samlReplyUrl": "", - "clientId": {{ tmpl.Exec "GenClientId" "CSGHub" | quote }}, - "clientSecret": {{ tmpl.Exec "GenClientSecret" "CSGHub" | quote }}, + "clientId": {{ generateClientId "CSGHub" | quote }}, + "clientSecret": {{ generateClientSecret "CSGHub" | quote }}, "providers": [ { "owner": "", @@ -362,7 +361,7 @@ "id_token" ], "redirectUris": [ - "{{ $csghub.external }}/api/v1/callback/casdoor" + "{{ $csghub.url }}/api/v1/callback/casdoor" ], "tokenFormat": "JWT", "tokenFields": [], @@ -378,7 +377,7 @@ "footerHtml": "2025 OpenCSG. All rights reserved.", "formCSS": "", "formOffset": 2, - "formSideHtml": "
", + "formSideHtml": "
", "failedSigninLimit": 5, "failedSigninFrozenTime": 15 }, @@ -386,8 +385,8 @@ "owner": "admin", "name": "Admin", "displayName": "Admin", - "logo": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/logo.png", - "homepageUrl": "{{ $csghub.external }}", + "logo": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/logo.png", + "homepageUrl": "{{ $csghub.url }}", "organization": "OpenCSG", "cert": "csghub-cert", "enablePassword": true, @@ -405,8 +404,8 @@ "tags": [ "is_admin" ], - "clientId": {{ tmpl.Exec "GenClientId" "Admin" | quote }}, - "clientSecret": {{ tmpl.Exec "GenClientSecret" "Admin" | quote }}, + "clientId": {{ generateClientId "Admin" | quote }}, + "clientSecret": {{ generateClientSecret "Admin" | quote }}, "providers": [ { "owner": "", @@ -592,7 +591,7 @@ "id_token" ], "redirectUris": [ - "{{ $csghub.external }}/-/temporal/auth/sso/callback" + "{{ $csghub.url }}/-/temporal/auth/sso/callback" ], "tokenFormat": "JWT", "tokenFields": [], @@ -608,17 +607,17 @@ "footerHtml": "2025 OpenCSG. All rights reserved.", "formCSS": "", "formOffset": 2, - "formSideHtml": "
", + "formSideHtml": "
", "failedSigninLimit": 5, "failedSigninFrozenTime": 15 } - {{- if $csgship.enable -}} + {{- if .csgship.enable }} ,{ "owner": "admin", "name": "CSGShip", "displayName": "CSGShip", - "logo": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/logo.png", - "homepageUrl": {{ tmpl.Exec "endpoint.csgship.frontend" . | quote }}, + "logo": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/logo.png", + "homepageUrl": {{ (resolveEndpoint $csghub.scheme $csghub.host .csgship.listen.frontend "127.0.0.1:8001").url | quote }}, "organization": "OpenCSG", "cert": "csgship-cert", "enablePassword": true, @@ -633,8 +632,8 @@ "enableLinkWithEmail": true, "orgChoiceMode": "Select", "samlReplyUrl": "", - "clientId": {{ tmpl.Exec "GenClientId" "CSGShip" | quote }}, - "clientSecret": {{ tmpl.Exec "GenClientSecret" "CSGShip" | quote }}, + "clientId": {{ generateClientId "CSGShip" | quote }}, + "clientSecret": {{ generateClientSecret "CSGShip" | quote }}, "providers": [ { "owner": "", @@ -841,7 +840,7 @@ "id_token" ], "redirectUris": [ - "{{ tmpl.Exec "endpoint.csgship.api" . }}/api/v1/account/casdoor/login/callback" + "{{ (resolveEndpoint $csghub.scheme $csghub.host .csgship.listen.api "127.0.0.1:8002").url }}/api/v1/account/casdoor/login/callback" ], "tokenFormat": "JWT", "tokenFields": [], @@ -857,11 +856,11 @@ "footerHtml": "2025 OpenCSG. All rights reserved.", "formCSS": "", "formOffset": 2, - "formSideHtml": "
", + "formSideHtml": "
", "failedSigninLimit": 5, "failedSigninFrozenTime": 15 } - {{- end -}} + {{- end }} ], "certs": [ { @@ -889,32 +888,32 @@ "privateKey": "-----BEGIN RSA PRIVATE KEY-----\nMIIJJwIBAAKCAgEAxu3H5b6lbk9sH30FtC1ZfSVof05DvYfP0/X1sNb0v3pSVcGe\nI9149wkoVvptsn5MvUZ1M4T9uNbvVAZzw/fA2CqwuTTWauSCHSiRui/pRH4LiE5U\n/631LoMhY3R63PI4NOWmRYR338VTaBWEhFvFoc1fSpBdOi3MCOzzQAW5meobeOkX\n/tI4HITt8U0dcEHtRM9KNWbcQlfeOwY1LoPti1cFmBgNvRjzmkEcBnFtpWnZbj1s\nwFZpQIosM6JOMqaKkn3o3QUpsHlDQP6+/Z42Wl1pRBdj2we1i3yP4+TjH7rU4p9I\npgXWdao7e7fepow0aJX8vfsgFdh4CZx4q3pWEwbaXnT6cbqNWewszSPhiddHdHLn\n4rhHwVd4GNwX99fFscW6V9ZSNmDyDPqY95nRYIyABc/LsuT3bYQzIVXfPqRx47TU\nhtdmP9NJBF+8juMquaA7XlMbIollOB8h3dxJ8KtIN+vYhO7+8+vgAfMXf3gzOHLq\nzfk3OJhQbj00MG0WKAu6ZC1mhrzeH5Nf6sPhisGaLly0XC1x1d8+q1e5Ua6p7Zfe\nv6qqT2FRtfSAvTTTCoP1k8U1dMA+QCfUnV+WOq/tDx/Cea1wF4wwIdwnolO3iPlH\nA+HLiujB1TniljfGpUtRi+theC8NPxeTLu06FoB4vYcgzqsP7BJcqpr1ancCAwEA\nAQKCAgBPgu06bDhNXht2fo9QFCxHtDMPZ2X2FJnCO/ZnJ5QOEMNmGYyvAy/9dcCU\n+2XrSUY9BTOFUyvivK5OZIHTG20OOgoHDRKDCTDknLmatNnFEahkZALfa1WrsKgg\ny4bsVziWFBCgirErVQDggnFEQjY9VQw4jnlk7rc1/hfASEmGNqeibXC03quk4pEE\nuPECVKeUsFIWBDYXom1mJDYPkJXa8v5+a2baZWXBCzM7dA0CZ8uLn6ufOSq6rD+8\np6wf3KdYpFD2PsEKE34JNYl7b8iD2ABiQgMcofkzdqdAzNTnctLyff4Ytg6hW1iJ\nO9YXtcSg5QpvmTea47ofBVbKG4zZOrC7w4IUpc2wu9ooAjN7WJLsMce7ic7Yu3Ig\n5ahKXbYnxTy5Dx2Yf2RfmVfAY+P0ED+Ia0cIFdyOELphokl2tddePgW/76r6AfF7\nX8kOatgdiS5dtNss+ddv0q95XL0qhTljGF8DqCz/8rjKBodPUnU4uoOzYqUgwl99\nCH9EQvzEbHjFxpfcuDX6/UaCBfyTv5OEj2Qf8+Gvg/xZbaTxsyffhjH4O9Sq6MJk\nW+D9VN4H7pKzWXrzk+LUhzHvMSyiNbTJb204DvS4aX+ZWHSfOqLa1Fh+/fcIrHZF\nHfH9rzv2Kv6zfb7d/RJudmbzViVxj+hTcxjDNdUBbmHUIHF7kQKCAQEAy44xejSP\nySED97NxwPycjk+KUz3UZG5ZzJ8Yob963up4gNECxkYVHkVOPTxJGd/jzlNAlj3y\n5pzeuTW8e9NrkSRSp2UAvuYSEQPBZX2svpGPjrS7GdC/DmBDLjdnTofVVGVGSHYV\n+pWl3RhnrKtCdoree7IAaYaOqYLUu5bpgPoiStSqFsgUVENEJpGHffu8Y2S+5i9/\n5/j6RkvsBVJG8fhTIoOKXlMWyTOVx+4ZGkQq7+3YyOep9URztQsmqQeUgx0ChOuP\nnHj3oJGzxkAdDlBfm82CH3lw/63vOjzhf3ogIc6o+HVLS2fZxch1bgbccND78v1Z\nGbJ4BW+uqrPjaQKCAQEA+i5utZ8ruDClzb7LY7hbeTRhqI2sJ2YFVQKmQtYgt1+D\noIR0Ndonwm/FteRj4LN1m1GtDHKT8EZKIsZoh5J0lqPAq/sUo3Y5LyKpoQZnb5mK\nsJcEm69TBdu6eAhRlzkUOxBnc88lewQqFikoY41rKnu6C5WWY9+UT3RKkjIrO7qI\nwZ5fc4h0dc+b6yKT+eNnk1nWhMVZIRxflUZjogwzKpp+YV2vsgc7lx4u46718Wus\naVOPC4bseMM0Edrxjo8ByFGEx5LA8LySaEMEKDMRX0zbt8NMZQ7eE4CTpAGTDNts\nuv3Gg+s/4OVTGEFKnPNps02KGBHsemgQZxPqCq+C3wKCAQAdI+AFZJzo6CcPQMsa\nG4+198EPpZxQPFPyrRvzm9vVKa/SkRlalRHU4YsPzR+R6rFMCjMGzxuE1BQ2Y3D2\nMRAHGTIT8KuoqHsRoH4BL3qZVXE9trXKPk37muBDL39EiMGLcWwcV3tSAjZabnH5\nEBNVsQ+kmZjhgnfN4ILfz/zhj+101Pj8hewTHAYXl2HUMUOI3xVNEqO267e4dIn3\nQQ4S+b+9AyO92Rs79nyMwJMrZXLYIi6UpEV6bTA7TLteG02xb2e665YFinO4suMP\nkIIYWH03arYkF1iL4bSLm8eUXkFXVaRu3SvIQbe1Trsw4x6Pr/tBj+8/dIMmFBTn\nLxVxAoIBAHyL9Q1xyiAOB9zfaP2h39f9Hcj9joCmp2zkvsRzOSJQ6IRWrEvK6RRA\n9JpSsu5AKQJDrPSPbdqmA25Ae/VSEbyUlOyJOIMP7fPrO6swNvo14/d5lw+yPpCi\nDLiI26V/AvVMe1QKAcy2DAWwQ60b0Af8RGhT9eSOf5T2fdE+LIu1lzNcloQ7oE6u\nJh7uveAfN4cBqGu8m1yCdYso2ZzomehRtSzfqQHl2+KlYg5zIXkoc1KUCJhieQK9\nmwRwjOS9iNOi0YlRI8A1AGXrRvyEP9fDuXZIQZXlyuTGk7mIFx7FdeG5SMSPruNV\nBwMCLhTokOx0Y5UMB0AA4nmOTms3aN0CggEAVjp0XLX+4RqmV/pEPMwC8ahL+zav\n8V039l8nozLTroIRL/X5OL2onS3jwqezWb9D8QrCXvbf3oU2xnLJkYWFVFeMJO7Z\nTHOAyyY2TfzWuzs3294DscElTbw+phQvz2nnuAmnsTnauNLweEyiXO8tFCcw2dst\ns2On4mhGPlMPitkBOWJQIHQAeW7AxTVe1eJGAttE9x57BMx5OdY7p0MpDZDKto8N\nyHGt624AVr+eSOZ9AHUk/3TdsoQBpTvrjb1fydQ8PSwMug+kYZ9lWsEnuEs9Ngdn\nJZXgXuCiZcYxF9Jn7JylCaaJToT4VXxfOey6iaPS1WC/DOS5DNu6hIlkWQ==\n-----END RSA PRIVATE KEY-----\n" } ], - {{- $initPassFile := "/etc/csghub/init_root_password" -}} - {{- $pass := "" -}} - {{- if file.Exists $initPassFile -}} - {{- $content := file.Read $initPassFile | trimSpace }} - {{- $parts := strings.Split ":" $content }} + {{- $initPassFile := "/etc/csghub/init_root_password" }} + {{- $pass := "" }} + {{- if fileExists $initPassFile }} + {{- $content := fileRead $initPassFile | trim }} + {{- $parts := splitList ":" $content }} {{- if gt (len $parts) 1 }} - {{- $pass = index $parts 1 -}} + {{- $pass = index $parts 1 }} {{- else }} - {{- $pass = $content -}} + {{- $pass = $content }} {{- end }} - {{- end -}} - {{- if not $pass -}} - {{- $pass = random.Alpha 24 -}} - {{- file.Write $initPassFile (printf "root:%s\n" $pass) -}} - {{- end -}} + {{- end }} + {{- if not $pass }} + {{- $pass = randomAlpha 24 }} + {{- fileWrite $initPassFile (printf "root:%s\n" $pass) }} + {{- end }} "users": [ { "owner": "OpenCSG", "name": "root", - "password": {{ $pass | crypto.Bcrypt | quote }}, + "password": {{ $pass | bcrypt | quote }}, "passwordSalt": "OpenCSG", "passwordType": "bcrypt", "displayName": "root", "email": "root@example.com", "type": "normal-user", - "avatar": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/robot.png", + "avatar": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/robot.png", "isAdmin": true, "signupApplication": "CSGHub", "tag": "is_admin" @@ -951,7 +950,7 @@ "port": 465, "disableSsl": false, "title": "OpenCSG Verification Code", - "content": "\u003c!DOCTYPE html\u003e\n\u003chtml lang=\u0022en\u0022\u003e\n\u003chead\u003e\n\u003cmeta charset=\u0022UTF-8\u0022\u003e\n\u003cmeta name=\u0022viewport\u0022 content=\u0022width=device-width, initial-scale=1.0\u0022\u003e\n\u003ctitle\u003eVerification Code Email\u003c/title\u003e\n\u003cstyle\u003ebody { font-family: Arial, sans-serif; } .email-container { width: 600px; margin: 0 auto; } .header { text-align: center; } .code { font-size: 24px; margin: 20px 0; text-align: center; } .footer { font-size: 12px; text-align: center; margin-top: 50px; } .footer a { color: #000; text-decoration: none; } .center { text-align: center; } .center a { text-decoration: none; color: #1C8B7F; }\u003c/style\u003e\n\u003c/head\u003e\n\u003cbody\u003e\n\u003cdiv class=\u0022email-container\u0022\u003e\n \u003cdiv class=\u0022header\u0022\u003e\n \u003cimg src=\u0022{{ $casdoorEndpoint }}/files/avatar/OpenCSG/logo.png\u0022 alt=\u0022OpenCSG Logo\u0022 width=\u0022300\u0022\u003e\n \u003c/div\u003e\n \u003cp\u003e\u003cstrong\u003e%{user.friendlyName}\u003c/strong\u003e, here is your verification code\u003c/p\u003e\n \u003cp\u003eUse this code for your transaction. It's valid for 5 minutes\u003c/p\u003e\n \u003cdiv class=\u0022code\u0022\u003e\n %s\n \u003c/div\u003e\n \u003chr\u003e\n \u003cp class=\u0022center\u0022\u003e\n \u003ca href=\u0022https://opencsg.com\u0022 target=\u0022_blank\u0022\u003eOpenCSG Team\u003c/a\u003e\n \u003c/p\u003e\n\u003c/div\u003e\n\u003c/body\u003e\n\u003c/html\u003e", + "content": "\u003c!DOCTYPE html\u003e\n\u003chtml lang=\u0022en\u0022\u003e\n\u003chead\u003e\n\u003cmeta charset=\u0022UTF-8\u0022\u003e\n\u003cmeta name=\u0022viewport\u0022 content=\u0022width=device-width, initial-scale=1.0\u0022\u003e\n\u003ctitle\u003eVerification Code Email\u003c/title\u003e\n\u003cstyle\u003ebody { font-family: Arial, sans-serif; } .email-container { width: 600px; margin: 0 auto; } .header { text-align: center; } .code { font-size: 24px; margin: 20px 0; text-align: center; } .footer { font-size: 12px; text-align: center; margin-top: 50px; } .footer a { color: #000; text-decoration: none; } .center { text-align: center; } .center a { text-decoration: none; color: #1C8B7F; }\u003c/style\u003e\n\u003c/head\u003e\n\u003cbody\u003e\n\u003cdiv class=\u0022email-container\u0022\u003e\n \u003cdiv class=\u0022header\u0022\u003e\n \u003cimg src=\u0022{{ $casdoorEp.url }}/files/avatar/OpenCSG/logo.png\u0022 alt=\u0022OpenCSG Logo\u0022 width=\u0022300\u0022\u003e\n \u003c/div\u003e\n \u003cp\u003e\u003cstrong\u003e%{user.friendlyName}\u003c/strong\u003e, here is your verification code\u003c/p\u003e\n \u003cp\u003eUse this code for your transaction. It's valid for 5 minutes\u003c/p\u003e\n \u003cdiv class=\u0022code\u0022\u003e\n %s\n \u003c/div\u003e\n \u003chr\u003e\n \u003cp class=\u0022center\u0022\u003e\n \u003ca href=\u0022https://opencsg.com\u0022 target=\u0022_blank\u0022\u003eOpenCSG Team\u003c/a\u003e\n \u003c/p\u003e\n\u003c/div\u003e\n\u003c/body\u003e\n\u003c/html\u003e", "enableSignAuthnRequest": false }, { @@ -961,12 +960,12 @@ "category": "OAuth", "type": "Custom", "method": "Normal", - "clientId": {{ $csgship.oauth.client_id | default "" | quote }}, - "clientSecret": {{ $csgship.oauth.client_secret | default "" | quote }}, - "customAuthUrl": {{ printf "%s/oauth/authorize" $csgship.oauth.issuer | default "" | quote }}, - "customTokenUrl": {{ printf "%s/oauth/token" $csgship.oauth.issuer | default "" | quote }}, - "customUserInfoUrl": {{ printf "%s/api/v4/user" $csgship.oauth.issuer | default "" | quote }}, - "customLogo": "{{ $casdoorEndpoint }}/files/avatar/OpenCSG/gitlab_favicon.png", + "clientId": {{ .csgship.oauth.client_id | default "" | quote }}, + "clientSecret": {{ .csgship.oauth.client_secret | default "" | quote }}, + "customAuthUrl": {{ printf "%s/oauth/authorize" .csgship.oauth.issuer | default "" | quote }}, + "customTokenUrl": {{ printf "%s/oauth/token" .csgship.oauth.issuer | default "" | quote }}, + "customUserInfoUrl": {{ printf "%s/api/v4/user" .csgship.oauth.issuer | default "" | quote }}, + "customLogo": "{{ $casdoorEp.url }}/files/avatar/OpenCSG/gitlab_favicon.png", "scopes": "read_user openid profile email", "userMapping": { @@ -976,8 +975,8 @@ "id": "id", "username": "username" }, - "issuerUrl": {{ $csgship.oauth.issuer | default "" | quote }}, - "providerUrl": {{ $csgship.oauth.issuer | default "" | quote }} + "issuerUrl": {{ .csgship.oauth.issuer | default "" | quote }}, + "providerUrl": {{ .csgship.oauth.issuer | default "" | quote }} }, { "owner": "OpenCSG", @@ -986,8 +985,8 @@ "category": "Storage", "type": "Local File System", "method": "Normal", - "domain": "{{ $casdoorEndpoint }}", - "pathPrefix": "/opt/csghub/etc/casdoor" + "domain": "{{ $casdoorEp.url }}", + "pathPrefix": "/opt/csghub/embedded/etc/casdoor" } ] -} \ No newline at end of file +} diff --git a/opt/csghub/etc/casdoor/token_jwt_key.pem b/opt/csghub/embedded/etc/casdoor/token_jwt_key.pem similarity index 100% rename from opt/csghub/etc/casdoor/token_jwt_key.pem rename to opt/csghub/embedded/etc/casdoor/token_jwt_key.pem diff --git a/opt/csghub/etc/consul/consul.hcl.sample b/opt/csghub/embedded/etc/consul/consul.hcl.sample similarity index 90% rename from opt/csghub/etc/consul/consul.hcl.sample rename to opt/csghub/embedded/etc/consul/consul.hcl.sample index 395924db..9a02c658 100755 --- a/opt/csghub/etc/consul/consul.hcl.sample +++ b/opt/csghub/embedded/etc/consul/consul.hcl.sample @@ -3,13 +3,12 @@ # Full configuration options can be found at https://www.consul.io/docs/agent/config -{{- $consul := (datasource "config").consul }} # datacenter # This flag controls the datacenter in which the agent is running. If not provided, # it defaults to "dc1". Consul has first-class support for multiple datacenters, but # it relies on proper configuration. Nodes in the same datacenter should be on a # single LAN. -datacenter = {{ $consul.datacenter | quote }} +datacenter = {{ .consul.datacenter | quote }} # data_dir # This flag provides a data directory for the agent to store state. This is required @@ -18,21 +17,21 @@ datacenter = {{ $consul.datacenter | quote }} # cluster state. Additionally, the directory must support the use of filesystem # locking, meaning some types of mounted folders (e.g. VirtualBox shared folders) may # not be suitable. -data_dir = {{ $consul.data_dir | quote }} +data_dir = {{ .consul.data_dir | quote }} # client_addr # The address to which Consul will bind client interfaces, including the HTTP and DNS # servers. By default, this is "127.0.0.1", allowing only loopback connections. In # Consul 1.0 and later this can be set to a space-separated list of addresses to bind # to, or a go-sockaddr template that can potentially resolve to multiple addresses. -client_addr = {{ $consul.client_addr | quote }} +client_addr = {{ .consul.client_addr | quote }} # ui # Enables the built-in web UI server and the required HTTP routes. This eliminates # the need to maintain the Consul web UI files separately from the binary. # Version 1.10 deprecated ui=true in favor of ui_config.enabled=true ui_config{ - enabled = {{ $consul.ui_config.enabled }} + enabled = {{ .consul.ui_config.enabled }} } # server @@ -44,21 +43,21 @@ ui_config{ # ensure availability in the case of node failure. Server nodes also participate in a # WAN gossip pool with server nodes in other datacenters. Servers act as gateways to # other datacenters and forward traffic as appropriate. -server = {{ $consul.server }} +server = {{ .consul.server }} # Bind addr # You may use IPv4 or IPv6 but if you have multiple interfaces you must be explicit. #bind_addr = "[::]" # Listen on all IPv6 -bind_addr = {{ $consul.bind_addr | quote }} +bind_addr = {{ .consul.bind_addr | quote }} # Advertise addr - if you want to point clients to a different address than bind or LB. -advertise_addr = {{ $consul.advertise_addr | quote }} +advertise_addr = {{ .consul.advertise_addr | quote }} # Enterprise License # As of 1.10, Enterprise requires a license_path and does not have a short trial. #license_path = "/etc/consul.d/consul.hclic" -{{- if $consul.server }} +{{- if .consul.server }} # bootstrap_expect # This flag provides the number of expected servers in the datacenter. Either this value # should not be provided or the value must agree with other servers in the cluster. When @@ -66,7 +65,7 @@ advertise_addr = {{ $consul.advertise_addr | quote }} # bootstraps the cluster. This allows an initial leader to be elected automatically. # This cannot be used in conjunction with the legacy -bootstrap flag. This flag requires # -server mode. -bootstrap_expect={{ $consul.bootstrap_expect }} +bootstrap_expect={{ .consul.bootstrap_expect }} {{- end }} # encrypt @@ -99,8 +98,8 @@ bootstrap_expect={{ $consul.bootstrap_expect }} #retry_join = ["provider=aws tag_key=... tag_value=..."] #retry_join = ["provider=azure tag_name=... tag_value=... tenant_id=... client_id=... subscription_id=... secret_access_key=..."] #retry_join = ["provider=gce project_name=... tag_value=..."] -{{- if $consul.retry_join }} -retry_join = [{{- range $i, $v := $consul.retry_join }}{{ if $i }}, {{ end }}{{ $v | quote }}{{- end }}] +{{- if .consul.retry_join }} +retry_join = [{{- range $i, $v := .consul.retry_join }}{{ if $i }}, {{ end }}{{ $v | quote }}{{- end }}] {{- else }} retry_join = ["127.0.0.1"] -{{- end }} \ No newline at end of file +{{- end }} diff --git a/opt/csghub/etc/csghub/templates/system/csghub-runsvdir.service b/opt/csghub/embedded/etc/csghub/csghub-runsvdir.service similarity index 100% rename from opt/csghub/etc/csghub/templates/system/csghub-runsvdir.service rename to opt/csghub/embedded/etc/csghub/csghub-runsvdir.service diff --git a/opt/csghub/etc/csghub/default.yaml b/opt/csghub/embedded/etc/csghub/default.yaml similarity index 99% rename from opt/csghub/etc/csghub/default.yaml rename to opt/csghub/embedded/etc/csghub/default.yaml index b176b286..aa548d3d 100644 --- a/opt/csghub/etc/csghub/default.yaml +++ b/opt/csghub/embedded/etc/csghub/default.yaml @@ -80,7 +80,7 @@ server: ## Casdoor (authentication) Integration casdoor: endpoint: "" ## Casdoor endpoint - certificate: "/opt/csghub/etc/casdoor/token_jwt_key.pem" ## SSL cert path + certificate: "/opt/csghub/embedded/etc/casdoor/token_jwt_key.pem" ## SSL cert path organization_name: "OpenCSG" ## Organization name in Casdoor application_name: "CSGHub" ## App name in Casdoor diff --git a/opt/csghub/etc/csghub/scripts/pg_executor b/opt/csghub/embedded/etc/csghub/scripts/pg_executor similarity index 100% rename from opt/csghub/etc/csghub/scripts/pg_executor rename to opt/csghub/embedded/etc/csghub/scripts/pg_executor diff --git a/opt/csghub/etc/csghub/scripts/pg_exists b/opt/csghub/embedded/etc/csghub/scripts/pg_exists similarity index 100% rename from opt/csghub/etc/csghub/scripts/pg_exists rename to opt/csghub/embedded/etc/csghub/scripts/pg_exists diff --git a/opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample b/opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample similarity index 99% rename from opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample rename to opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample index 0fa5ad6b..3b0edc0d 100644 --- a/opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample +++ b/opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample @@ -74,7 +74,7 @@ csghub: ## Casdoor (authentication) Integration # casdoor: # endpoint: "http://csghub.example.com:8000" ## Casdoor endpoint - # certificate: "/opt/csghub/etc/casdoor/token_jwt_key.pem" ## SSL cert path + # certificate: "/opt/csghub/embedded/etc/casdoor/token_jwt_key.pem" ## SSL cert path # organization_name: "OpenCSG" ## Organization name in Casdoor # application_name: "CSGHub" ## App name in Casdoor diff --git a/opt/csghub/embedded/etc/gitaly/config.toml.sample b/opt/csghub/embedded/etc/gitaly/config.toml.sample new file mode 100755 index 00000000..eaf2faac --- /dev/null +++ b/opt/csghub/embedded/etc/gitaly/config.toml.sample @@ -0,0 +1,148 @@ +# Example Gitaly configuration file with all configuration blocks dynamically defined. + +{{- $csghub := resolveUrl .csghub.external_url }} +{{- $dataDir := .gitaly.data_dir | default "/var/opt/csghub/gitaly" }} +{{- $listenAddr := .gitaly.listen_addr | default "" }} +{{- if eq $listenAddr "" }} +# Unix socket path for Gitaly +socket_path = {{ .gitaly.socket_path | default (printf "%s/gitaly.socket" $dataDir) | quote }} +{{- end }} + +# Directory containing Gitaly executables +bin_dir = {{ .gitaly.bin_dir | quote }} + +# Runtime directory for Gitaly +runtime_dir = {{ printf "%s/run" $dataDir | quote }} + +# Unencrypted TCP listen address +listen_addr = {{ .gitaly.listen_addr | quote }} + +{{- if and .gitaly.tls_listen_addr (not .gitaly.listen_addr) }} +# TCP with TLS listen address +tls_listen_addr = {{ .gitaly.tls_listen_addr | quote }} +{{- end }} + +# Prometheus metrics listen address +prometheus_listen_addr = {{ .gitaly.prometheus_listen_addr | quote }} + +[auth] +# Auth token required for Gitaly authentication +token = {{ .gitaly.auth.token | default (pbkdf2 "gitaly" "opencsg" 2048 8) | quote }} +transitioning = {{ .gitaly.auth.transitioning }} + +{{- if has .gitaly "tls" }} +# Gitaly supports TLS encryption. You must bring your own certificates because this isn’t provided automatically. +[tls] +{{- if .gitaly.tls.certificate_path }} +certificate_path = {{ .gitaly.tls.certificate_path | default "/var/opt/csghub/gitaly/ssl/cert.cert" | quote }} +{{- end }} +{{- if .gitaly.tls.key_path }} +key_path = {{ .gitaly.tls.key_path | default "/var/opt/csghub/gitaly/ssl/key.pem" | quote }} +{{- end }} +{{- end }} + +# Git settings +[git] +use_bundled_binaries = {{ .gitaly.git.use_bundled_binaries }} +ignore_gitconfig = {{ .gitaly.git.ignore_gitconfig }} +bin_path = {{ .gitaly.git.bin_path | quote }} +catfile_cache_size = {{ .gitaly.git.catfile_cache_size }} +signing_key = "{{ .gitaly.git.signing_key }}" + + # Git configuration + {{- range $config := .gitaly.git.config }} + [[git.config]] + key = {{ $config.key | quote }} + value = {{ $config.value | quote }} + {{- end }} + +# Storages configuration +{{- range .gitaly.storage }} +[[storage]] +name = {{ .name | quote }} +path = {{ .path | default (printf "%s/repositories" $dataDir) | quote }} +{{- end }} + +[logging] +format = {{ .gitaly.logging.format | quote }} +level = {{ .gitaly.logging.level | quote }} +dir = {{ .gitaly.logging.dir | quote }} +sentry_dsn = {{ .gitaly.logging.sentry_dsn | default "" | quote }} +sentry_environment = {{ .gitaly.logging.sentry_environment | default "" | quote }} + +[prometheus] +grpc_latency_buckets = [{{- range $i, $v := .gitaly.prometheus.grpc_latency_buckets }}{{if $i}}, {{end}}{{$v}}{{- end }}] + +[hooks] +custom_hooks_dir = {{ .gitaly.hooks.custom_hooks_dir | default (printf "%s/custom_hooks" $dataDir) | quote }} + +[gitlab] +url = {{ $csghub.url | quote }} +relative_url_root = {{ .csghub.relative_url_root | default "/" | quote }} +{{- if .gitaly.secret_file }} +secret_file = {{ .gitaly.secret_file | quote }} +{{- else }} +secret = {{ .gitaly.secret | default (persistentSeed | sha256sum) | quote }} +{{- end }} + +[gitlab.http-settings] +read_timeout = {{ .gitaly.http_settings.read_timeout }} +self_signed_cert = {{ .gitaly.http_settings.self_signed_cert }} + +{{- range $concurrency := .gitaly.concurrency }} +[[concurrency]] +{{- range $key, $value := $concurrency }} +{{- if isKind "string" $value }} +{{ $key }} = {{ $value | quote }} +{{- else if isKind "number" $value }} +{{ $key }} = {{ $value }} +{{- end }} +{{- end }} +{{- end }} +{{- range $limiting := .gitaly.rate_limiting }} +[[rate_limiting]] +{{- range $key, $value := $limiting }} +{{- if isKind "string" $value }} +{{ $key }} = {{ $value | quote }} +{{- else if isKind "number" $value }} +{{ $key }} = {{ $value }} +{{- end }} +{{- end }} +{{- end }} + +[daily_maintenance] +disabled = {{ .gitaly.daily_maintenance.disable }} +start_hour = {{ .gitaly.daily_maintenance.start_hour }} +start_minute = {{ .gitaly.daily_maintenance.start_minute }} +duration = {{ .gitaly.daily_maintenance.duration | quote }} +storages = [{{- range $i, $v := .gitaly.daily_maintenance.storages }}{{if $i}}, {{end}}{{ $v | quote }}{{- end }}] + +{{- if has .gitaly "cgroups" }} +[cgroups] +mountpoint = {{ .gitaly.cgroups.mountpoint | default "/sys/fs/cgroup" | quote }} +hierarchy_root = {{ .gitaly.cgroups.hierarchy_root | default "gitaly" | quote }} +memory_bytes = {{ .gitaly.cgroups.memory_bytes | default 64424509440 }} +cpu_shares = {{ .gitaly.cgroups.cpu_shares | default 1024 }} +cpu_quota_us = {{ .gitaly.cgroups.cpu_quota_us | default 400000 }} +[cgroups.repositories] +count = {{ .gitaly.cgroups.repositories.count | default 500 }} +memory_bytes = {{ .gitaly.cgroups.repositories.memory_bytes | default 12884901888 }} +cpu_shares = {{ .gitaly.cgroups.repositories.cpu_shares | default 512 }} +cpu_quota_us = {{ .gitaly.cgroups.repositories.cpu_quota_us | default 200000 }} +{{- end }} +{{- if has .gitaly "backup" }} +[backup] +go_cloud_url = {{ .gitaly.backup.go_cloud_url | default "gs://gitaly-backups" | quote }} +layout = {{ .gitaly.backup.layout | default "pointer" | quote }} +wal_backup_go_cloud_url = {{ .gitaly.backup.wal_backup_go_cloud_url | default "gs://gitaly-wal-backups" | quote }} +wal_backup_worker_count = {{ .gitaly.backup.wal_backup_worker_count | default 1 }} +buffer_size = {{ .gitaly.backup.buffer_size | default 0 }} +{{- end }} +{{- if has .gitaly "bundle_uri" }} +[bundle_uri] +go_cloud_url = {{ .gitaly.bundle_uri.go_cloud_url | default "gs://my-bundle-uri-bucket" | quote }} +{{- end }} + +[timeout] +upload_pack_negotiation = {{ .gitaly.timeout.upload_pack_negotiation | default "10m" | quote }} +upload_archive_negotiation = {{ .gitaly.timeout.upload_archive_negotiation | default "1m" | quote }} diff --git a/opt/csghub/etc/gitlab_shell/config.yml.sample b/opt/csghub/embedded/etc/gitlab_shell/config.yml.sample similarity index 92% rename from opt/csghub/etc/gitlab_shell/config.yml.sample rename to opt/csghub/embedded/etc/gitlab_shell/config.yml.sample index b2ce5cd0..a078bdde 100644 --- a/opt/csghub/etc/gitlab_shell/config.yml.sample +++ b/opt/csghub/embedded/etc/gitlab_shell/config.yml.sample @@ -2,9 +2,9 @@ # If you change this file in a Merge Request, please also create # a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests # -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -{{- $gitlabShell := (datasource "config").gitlab_shell -}} -{{- $dataDir := $gitlabShell.data | default "/var/opt/csghub/gitlab_shell" }} + +{{- $dataDir := .gitlab_shell.data | default "/var/opt/csghub/gitlab_shell" }} +{{- $csghub := resolveUrl .csghub.external_url }} # GitLab user. git by default user: git @@ -15,7 +15,7 @@ user: git # only listen on a Unix domain socket. For Unix domain sockets use # "http+unix://", e.g. # "http+unix://%2Fpath%2Fto%2Fsocket" -gitlab_url: {{ $csghub.external }} +gitlab_url: {{ $csghub.url }} # When a http+unix:// is used in gitlab_url, this is the relative URL root to GitLab. # Not used if gitlab_url is http:// or https://. @@ -39,7 +39,7 @@ auth_file: "/home/git/.ssh/authorized_keys" # File that contains the secret key for verifying access to GitLab. # Default is .gitlab_shell_secret in the gitlab-shell directory. -secret_file: "/opt/csghub/etc/gitlab_shell/.gitlab_shell_secret" +secret_file: "/opt/csghub/service/gitlab_shell/.gitlab_shell_secret" # The secret field supersedes the secret_file, and if set that # file will not be read. @@ -47,9 +47,9 @@ secret: "signing-key" # Log file. # Default is gitlab-shell.log in the root directory. -{{- $log := $gitlabShell.log }} +{{- $log := .gitlab_shell.log }} {{- $logDir := $log.dir }} -log_file: {{ filepath.Join $logDir "gitlab-shell.log" }} +log_file: {{ filepathJoin $logDir "gitlab-shell.log" }} # Log level. INFO by default log_level: {{ $log.level }} @@ -69,7 +69,7 @@ audit_usernames: false # This section configures the built-in SSH server. Ignored when running on OpenSSH. sshd: # Address which the SSH server listens on. Defaults to [::]:22. - listen: {{ printf "[::]:%d" $gitlabShell.ssh_port | quote }} + listen: {{ printf "[::]:%d" .gitlab_shell.ssh_port | quote }} # Set to true if gitlab-sshd is being fronted by a load balancer that implements # the PROXY protocol. proxy_protocol: false diff --git a/opt/csghub/etc/loki/loki-config.yaml.sample b/opt/csghub/embedded/etc/loki/loki-config.yaml.sample similarity index 80% rename from opt/csghub/etc/loki/loki-config.yaml.sample rename to opt/csghub/embedded/etc/loki/loki-config.yaml.sample index 151601d2..6eef77b7 100644 --- a/opt/csghub/etc/loki/loki-config.yaml.sample +++ b/opt/csghub/embedded/etc/loki/loki-config.yaml.sample @@ -1,11 +1,10 @@ -{{- $loki := (datasource "config").loki }} -{{- $dataDir := (datasource "config").loki.data | default "/var/opt/csghub/loki" -}} +{{- $dataDir := .loki.data | default "/var/opt/csghub/loki" }} auth_enabled: false server: - http_listen_port: {{ $loki.listen_port | default 3100 }} - log_level: {{ $loki.log_level | default "info" }} + http_listen_port: {{ .loki.listen_port | default 3100 }} + log_level: {{ .loki.log_level | default "info" }} common: instance_addr: 127.0.0.1 @@ -43,4 +42,4 @@ ruler: {} # # If you would like to disable reporting, uncomment the following lines: #analytics: -# reporting_enabled: false \ No newline at end of file +# reporting_enabled: false diff --git a/opt/csghub/etc/nats/nats.conf.sample b/opt/csghub/embedded/etc/nats/nats.conf.sample similarity index 78% rename from opt/csghub/etc/nats/nats.conf.sample rename to opt/csghub/embedded/etc/nats/nats.conf.sample index dd0b23dc..03157b2c 100644 --- a/opt/csghub/etc/nats/nats.conf.sample +++ b/opt/csghub/embedded/etc/nats/nats.conf.sample @@ -1,8 +1,7 @@ -{{- $nats := (datasource "config").nats }} -{{- $dataDir := $nats.data }} -{{- $auth := $nats.auth }} +{{- $dataDir := .nats.data }} +{{- $auth := .nats.auth }} {{- $natsUser := $auth.user }} -{{- $natsPassword := $auth.password | default (crypto.PBKDF2 $natsUser "opencsg" 2048 8) }} +{{- $natsPassword := $auth.password | default (pbkdf2 $natsUser "opencsg" 2048 8) }} debug = false trace = false @@ -27,7 +26,7 @@ authorization { } users = [ - {user: {{ $auth.user }}, password: {{ $natsPassword | crypto.Bcrypt }}, permissions: $ADMIN} + {user: {{ $auth.user }}, password: {{ $natsPassword | bcrypt }}, permissions: $ADMIN} ] } diff --git a/opt/csghub/etc/nginx/html/403.html.sample b/opt/csghub/embedded/etc/nginx/html/403.html.sample similarity index 100% rename from opt/csghub/etc/nginx/html/403.html.sample rename to opt/csghub/embedded/etc/nginx/html/403.html.sample diff --git a/opt/csghub/etc/nginx/html/404.html.sample b/opt/csghub/embedded/etc/nginx/html/404.html.sample similarity index 100% rename from opt/csghub/etc/nginx/html/404.html.sample rename to opt/csghub/embedded/etc/nginx/html/404.html.sample diff --git a/opt/csghub/etc/nginx/html/500.html.sample b/opt/csghub/embedded/etc/nginx/html/500.html.sample similarity index 100% rename from opt/csghub/etc/nginx/html/500.html.sample rename to opt/csghub/embedded/etc/nginx/html/500.html.sample diff --git a/opt/csghub/etc/nginx/html/502.html.sample b/opt/csghub/embedded/etc/nginx/html/502.html.sample similarity index 100% rename from opt/csghub/etc/nginx/html/502.html.sample rename to opt/csghub/embedded/etc/nginx/html/502.html.sample diff --git a/opt/csghub/etc/nginx/html/503.html.sample b/opt/csghub/embedded/etc/nginx/html/503.html.sample similarity index 100% rename from opt/csghub/etc/nginx/html/503.html.sample rename to opt/csghub/embedded/etc/nginx/html/503.html.sample diff --git a/opt/csghub/etc/nginx/nginx.conf.sample b/opt/csghub/embedded/etc/nginx/nginx.conf.sample similarity index 88% rename from opt/csghub/etc/nginx/nginx.conf.sample rename to opt/csghub/embedded/etc/nginx/nginx.conf.sample index 24045618..8dde76b0 100644 --- a/opt/csghub/etc/nginx/nginx.conf.sample +++ b/opt/csghub/embedded/etc/nginx/nginx.conf.sample @@ -11,7 +11,7 @@ events { use epoll; # Use epoll for scalable I/O } -{{ $nginx := (datasource "config").nginx }} +{{- $csghub := resolveUrl .csghub.external_url }} # HTTP block configuration http { # Disable server version token in responses @@ -96,20 +96,17 @@ http { "~^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" 1; } - {{- if $nginx.rate_limit.enable }} - limit_req_zone $binary_remote_addr zone=global:10m rate={{ $nginx.rate_limit.request }}; + {{- if .nginx.rate_limit.enable }} + limit_req_zone $binary_remote_addr zone=global:10m rate={{ .nginx.rate_limit.request }}; limit_conn_zone $binary_remote_addr zone=addr:10m; {{- end }} - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} - {{- $server := (datasource "config").server -}} - {{- $nginx := (datasource "config").nginx -}} - {{- $dataDir := $nginx.data }} + {{- $dataDir := .nginx.data }} - {{- $csghubCert := $nginx.ssl_certificate | default (printf "/etc/csghub/ssl/%s.crt" $csghub.host) | quote }} - {{- $csghubCertKey := $nginx.ssl_certificate_key | default (printf "/etc/csghub/ssl/%s.key" $csghub.host) | quote }} + {{- $csghubCert := .nginx.ssl_certificate | default (printf "/etc/csghub/ssl/%s.crt" $csghub.host) | quote }} + {{- $csghubCertKey := .nginx.ssl_certificate_key | default (printf "/etc/csghub/ssl/%s.key" $csghub.host) | quote }} - {{- if eq $csghub.scheme "https" -}} + {{- if eq $csghub.scheme "https" }} # Main server configurations server { listen 80; @@ -135,9 +132,9 @@ http { ssl_certificate_key {{ $csghubCertKey }}; {{- end }} - {{- if $nginx.rate_limit.enable }} - limit_req zone=global burst={{ $nginx.rate_limit.burst }}; - limit_conn addr {{ $nginx.rate_limit.conn }}; + {{- if .nginx.rate_limit.enable }} + limit_req zone=global burst={{ .nginx.rate_limit.burst }}; + limit_conn addr {{ .nginx.rate_limit.conn }}; {{- end }} # Default route @@ -180,10 +177,9 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } - {{- $temporalUI := (datasource "config").temporal_ui -}} - {{ if $temporalUI.enable }} + {{- if .temporal_ui.enable }} # Temporal UI route with basic auth - location {{ $temporalUI.public_path }} { + location {{ .temporal_ui.public_path }} { proxy_pass http://127.0.0.1:8182; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -224,8 +220,7 @@ http { proxy_read_timeout 900; } - {{- $runner := (datasource "config").runner -}} - {{- if and (not $runner.use_public_domain) (regexp.Match `^[a-zA-Z0-9.-]+$` $csghub.host) }} + {{- if and (not .runner.use_public_domain) (regexMatch `^[a-zA-Z0-9.-]+$` $csghub.host) }} # Endpoint route location /endpoint/ { proxy_pass http://127.0.0.1:8083; @@ -260,9 +255,9 @@ http { } } - {{ if and $runner.use_public_domain - (regexp.Match `^[a-zA-Z0-9.-]+$` $csghub.host) - (not (regexp.Match `^[0-9.]+$` $csghub.host)) -}} + {{ if and .runner.use_public_domain + (regexMatch `^[a-zA-Z0-9.-]+$` $csghub.host) + (not (regexMatch `^[0-9.]+$` $csghub.host)) -}} server { {{- if eq $csghub.scheme "https" }} listen 443 ssl; @@ -276,9 +271,9 @@ http { ssl_certificate_key {{ $csghubCertKey }}; {{- end }} - {{- if $nginx.rate_limit.enable }} - limit_req zone=global burst={{ $nginx.rate_limit.burst }}; - limit_conn addr {{ $nginx.rate_limit.conn }}; + {{- if .nginx.rate_limit.enable }} + limit_req zone=global burst={{ .nginx.rate_limit.burst }}; + limit_conn addr {{ .nginx.rate_limit.conn }}; {{- end }} # Endpoint route @@ -306,23 +301,23 @@ http { } {{- end }} - {{ $casdoor := tmpl.Exec "config.casdoor.conn" . | data.YAML -}} # Casdoor service configuration + {{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} server { {{- if eq $csghub.scheme "https" }} - listen {{ $casdoor.port }} ssl; - server_name {{ $csghub.host }}; + listen {{ $casdoorEp.port }} ssl; + server_name {{ $casdoorEp.host }}; ssl_certificate {{ $csghubCert }}; ssl_certificate_key {{ $csghubCertKey }}; {{- else }} - listen {{ $casdoor.port }}; + listen {{ $casdoorEp.port }}; server_name {{ $csghub.host }}; {{- end }} - {{- if $nginx.rate_limit.enable }} - limit_req zone=global burst={{ $nginx.rate_limit.burst }}; - limit_conn addr {{ $nginx.rate_limit.conn }}; + {{- if .nginx.rate_limit.enable }} + limit_req zone=global burst={{ .nginx.rate_limit.burst }}; + limit_conn addr {{ .nginx.rate_limit.conn }}; {{- end }} location / { @@ -402,4 +397,4 @@ http { proxy_pass http://127.0.0.1:9001; } } -} \ No newline at end of file +} diff --git a/opt/csghub/etc/patroni/patroni.yaml.sample b/opt/csghub/embedded/etc/patroni/patroni.yaml.sample similarity index 56% rename from opt/csghub/etc/patroni/patroni.yaml.sample rename to opt/csghub/embedded/etc/patroni/patroni.yaml.sample index 222c218a..91be8da5 100755 --- a/opt/csghub/etc/patroni/patroni.yaml.sample +++ b/opt/csghub/embedded/etc/patroni/patroni.yaml.sample @@ -1,25 +1,23 @@ -{{- $patroni := (datasource "config").patroni }} -scope: {{ $patroni.scope | squote }} -namespace: {{ $patroni.namespace | squote }} -name: {{ $patroni.name | default (env.Getenv "HOSTNAME") }} +scope: {{ .patroni.scope | squote }} +namespace: {{ .patroni.namespace | squote }} +name: {{ .patroni.name | default (env "HOSTNAME") }} log: format: '%(asctime)s %(levelname)s: %(message)s' - level: {{ $patroni.log.level }} + level: {{ .patroni.log.level }} max_queue_size: 1000 traceback_level: ERROR - type: {{ $patroni.log.type }} + type: {{ .patroni.log.type }} restapi: - connect_address: {{ printf "%s:8008" $patroni.listen }} - listen: {{ printf "%s:8008" $patroni.listen }} + connect_address: {{ printf "%s:8008" .patroni.listen }} + listen: {{ printf "%s:8008" .patroni.listen }} consul: - host: {{ $patroni.consul }} + host: {{ .patroni.consul }} + +{{- $cpuCores := .resources.cpu_cores }} -{{- $cpuCores := (datasource "resources").cpu_cores }} -{{- $memory := (datasource "resources").memory_mb }} -{{- $disk := (datasource "resources").disk_gb }} # The bootstrap configuration. Works only when the cluster is not yet initialized. # If the cluster is already initialized, all changes in the `bootstrap` section are ignored! bootstrap: @@ -28,8 +26,8 @@ bootstrap: # WARNING! If you want to change any of the parameters that were set up # via `bootstrap.dcs` section, please use `patronictl edit-config`! dcs: - {{- if has $patroni "bootstrap" }} - {{- $bootstrap := $patroni.bootstrap }} + {{- if has .patroni "bootstrap" }} + {{- $bootstrap := .patroni.bootstrap }} loop_wait: {{ $bootstrap.loop_wait }} retry_timeout: {{ $bootstrap.retry_timeout }} ttl: {{ $bootstrap.ttl }} @@ -41,33 +39,33 @@ bootstrap: {{- end }} postgresql: parameters: - {{- $postgresql := $patroni.postgresql.parameters }} + {{- $postgresql := .patroni.postgresql.parameters }} {{- $sharedBuffers := $postgresql.shared_buffers }} - {{- $sharedBuffersWithoutUnit := regexp.Find "^\\d+" ($sharedBuffers | strings.ToUpper) }} - {{- $memoryAllocated := math.Mul (math.Div $sharedBuffersWithoutUnit 1024) 4 }} + {{- $sharedBuffersWithoutUnit := regexFind "^\\d+" ($sharedBuffers | upper) }} + {{- $memoryAllocated := mulf (div $sharedBuffersWithoutUnit 1024) 4 }} hot_standby: 'on' max_locks_per_transaction: 128 - max_prepared_transactions: {{ math.Floor (math.Mul (math.Div (math.Mul $memoryAllocated 1000) 4) 5) }} - max_worker_processes: {{ math.Max (math.Floor (math.Div $cpuCores 2)) 2 }} - {{- $maxConnections := math.Max (math.Floor (math.Div $memoryAllocated 20)) 100 }} + max_prepared_transactions: {{ floor (mulf (div (mul $memoryAllocated 1000) 4) 5) }} + max_worker_processes: {{ max (floor (div $cpuCores 2)) 2 }} + {{- $maxConnections := max (floor (div $memoryAllocated 20)) 100 }} max_connections: {{ $postgresql.max_connections | default $maxConnections }} superuser_reserved_connections: 5 - work_mem: {{ printf "%dMB" (conv.ToInt (math.Max (math.Min (math.Floor (math.Div (math.Mul $memory 1024) 4096)) 64) 16)) }} - maintenance_work_mem: {{ printf "%dMB" (conv.ToInt (math.Min 8192 (math.Floor (math.Div (math.Div (math.Mul $memory 1024) 8) (math.Min (math.Max (math.Floor (math.Div $cpuCores 2)) 2) 16))))) }} + work_mem: {{ printf "%dMB" (int (max (min (floor (div (mul .resources.memory_mb 1024) 4096)) 64) 16)) }} + maintenance_work_mem: {{ printf "%dMB" (int (min 8192 (floor (div (div (mul .resources.memory_mb 1024) 8) (min (max (floor (div $cpuCores 2)) 2) 16))))) }} bgwriter_delay: 10ms bgwriter_lru_maxpages: 1000 bgwriter_lru_multiplier: 10 bgwriter_flush_after: 521kB effective_io_concurrency: 0 - max_parallel_maintenance_workers: {{ math.Min (math.Max (math.Floor (math.Div $cpuCores 2)) 2) 16 }} - max_parallel_workers_per_gather: {{ math.Min (math.Max (math.Floor (math.Sub $cpuCores 4)) 2) 24 }} - max_parallel_workers: {{ math.Max (math.Floor (math.Sub $cpuCores 4)) 2 }} + max_parallel_maintenance_workers: {{ min (max (floor (div $cpuCores 2)) 2) 16 }} + max_parallel_workers_per_gather: {{ min (max (floor (sub $cpuCores 4)) 2) 24 }} + max_parallel_workers: {{ max (floor (sub $cpuCores 4)) 2 }} checkpoint_timeout: 10min checkpoint_completion_target: 0.9 random_page_cost: 1.1 - effective_cache_size: {{ printf "%dGB" (conv.ToInt (math.Floor (math.Mul (math.Div $memory 1024) 0.75))) }} + effective_cache_size: {{ printf "%dGB" (int (floor (mulf (div .resources.memory_mb 1024) 0.75))) }} log_min_duration_statement: {{ $postgresql.log_min_duration_statement | default "2s" }} - autovacuum_max_workers: {{ math.Max (math.Min (math.Floor (math.Div $cpuCores 2)) 8) 5 }} + autovacuum_max_workers: {{ max (min (floor (div $cpuCores 2)) 8) 5 }} autovacuum_vacuum_scale_factor: 0.02 autovacuum_analyze_scale_factor: 0.01 autovacuum_freeze_max_age: 200000000 @@ -89,7 +87,7 @@ bootstrap: initdb: - encoding: UTF8 - data-checksums - {{- $standby := $patroni.standby_leader }} + {{- $standby := .patroni.standby_leader }} {{- $standbyEnabled := $standby.enable }} {{- if $standbyEnabled }} standby_leader: @@ -99,45 +97,44 @@ bootstrap: recovery_min_apply_delay: {{ $standby.recovery_min_apply_delay }} {{- end }} -{{ $postgresql := $patroni.postgresql }} +{{- $postgresql := .patroni.postgresql }} postgresql: {{- $auth := $postgresql.authentication }} authentication: superuser: username: postgres - password: {{ $auth.postgres | default (crypto.PBKDF2 "postgres" "opencsg" 2048 8) | squote }} + password: {{ $auth.postgres | default (pbkdf2 "postgres" "opencsg" 2048 8) | squote }} replication: username: replicator - password: {{ $auth.replicator | default (crypto.PBKDF2 "replicator" "opencsg" 2048 8) | squote }} + password: {{ $auth.replicator | default (pbkdf2 "replicator" "opencsg" 2048 8) | squote }} rewind: username: rewinder - password: {{ $auth.rewinder | default (crypto.PBKDF2 "rewinder" "opencsg" 2048 8) | squote }} + password: {{ $auth.rewinder | default (pbkdf2 "rewinder" "opencsg" 2048 8) | squote }} bin_dir: /opt/csghub/embedded/sv/postgresql/bin data_dir: {{ $postgresql.data_dir }} - connect_address: {{ printf "%s:5432" $patroni.listen }} - listen: {{ printf "%s:5432" $patroni.listen }} + connect_address: {{ printf "%s:5432" .patroni.listen }} + listen: {{ printf "%s:5432" .patroni.listen }} pgpass: {{ $postgresql.pgpass }} parameters: password_encryption: scram-sha-256 - remove_data_directory_on_rewind_failure: {{ $patroni.remove_data_directory_on_rewind_failure }} - remove_data_directory_on_diverged_timelines: {{ $patroni.remove_data_directory_on_diverged_timelines }} - {{- if $patroni.callback.enable }} + remove_data_directory_on_rewind_failure: {{ .patroni.remove_data_directory_on_rewind_failure }} + remove_data_directory_on_diverged_timelines: {{ .patroni.remove_data_directory_on_diverged_timelines }} + {{- if .patroni.callback.enable }} callbacks: - on_start: /var/opt/csghub/patroni/patroni_callback.sh - on_stop: /var/opt/csghub/patroni/patroni_callback.sh - on_role_change: /var/opt/csghub/patroni/patroni_callback.sh + on_start: /opt/csghub/service/patroni/etc/patroni_callback.sh + on_stop: /opt/csghub/service/patroni/etc/patroni_callback.sh + on_role_change: /opt/csghub/service/patroni/etc/patroni_callback.sh {{- end }} -{{- if has $patroni "watchdog" }} -{{- $watchdog := $patroni.watchdog }} +{{- if has .patroni "watchdog" }} +{{- $watchdog := .patroni.watchdog }} watchdog: mode: {{ $watchdog.mode | default "automatic" }} device: /dev/watchdog safety_margin: {{ $watchdog.safety_margin | default 20 }} {{- end }} - -{{- if has $patroni "tags" }} -{{- $tags := $patroni.tags }} +{{- if has .patroni "tags" }} +{{- $tags := .patroni.tags }} tags: {{- if $tags.clonefrom }} clonefrom: {{ $tags.clonefrom }} diff --git a/opt/csghub/etc/patroni/patroni_callback.sh.sample b/opt/csghub/embedded/etc/patroni/patroni_callback.sh.sample similarity index 73% rename from opt/csghub/etc/patroni/patroni_callback.sh.sample rename to opt/csghub/embedded/etc/patroni/patroni_callback.sh.sample index 4174e73b..030540e7 100755 --- a/opt/csghub/etc/patroni/patroni_callback.sh.sample +++ b/opt/csghub/embedded/etc/patroni/patroni_callback.sh.sample @@ -4,19 +4,18 @@ readonly OPERATION=$1 readonly ROLE=$2 readonly SCOPE=$3 -{{- if has (datasource "config").patroni "callback" }} -{{- $callback := (datasource "config").patroni.callback }} -VIP={{ $callback.virtual_ip | default "127.0.0.1" }} -PREFIX={{ $callback.virtual_ip | default 24 }} -BRD={{ $parts := strings.Split "." $callback.virtual_ip }}{{ join (coll.Slice $parts 0 3) "." }}.255 -INF={{ $callback.interface | default "eth0" }} +{{- if has .patroni "callback" }} + +VIP={{ .patroni.callback.virtual_ip | default "127.0.0.1" }} +PREFIX={{ .patroni.callback.virtual_ip | default 24 }} +BRD={{ $parts := splitList "." .patroni.callback.virtual_ip }}{{ join (sliceList $parts 0 3) "." }}.255 +INF={{ .patroni.callback.interface | default "eth0" }} {{- end }} function usage() { echo "Usage: $0 "; exit 1; - echo "$(date "+%Y-%m-%d %H:%M:%S %z") This is patroni callback $OPERATION $ROLE $SCOPE" case $OPERATION in diff --git a/opt/csghub/etc/postgresql/pg_hba.conf.sample b/opt/csghub/embedded/etc/postgresql/pg_hba.conf.sample similarity index 97% rename from opt/csghub/etc/postgresql/pg_hba.conf.sample rename to opt/csghub/embedded/etc/postgresql/pg_hba.conf.sample index 90337778..6176408b 100755 --- a/opt/csghub/etc/postgresql/pg_hba.conf.sample +++ b/opt/csghub/embedded/etc/postgresql/pg_hba.conf.sample @@ -82,7 +82,6 @@ # the database superuser. If you do not trust all your local users, # use another authentication method. - # TYPE DATABASE USER ADDRESS METHOD # "local" is for Unix domain socket connections only @@ -96,9 +95,9 @@ host all all ::1/128 trust local replication all trust host replication all 127.0.0.1/32 trust host replication all ::1/128 trust -{{- $postgresql := (datasource "config").postgresql }} -{{- if has $postgresql "pg_hba" }} -{{- range (datasource "config").postgresql.pg_hba }} + +{{- if has .postgresql "pg_hba" }} +{{- range .postgresql.pg_hba }} {{ . }} {{- end }} {{- end }} diff --git a/opt/csghub/etc/postgresql/postgresql.conf.sample b/opt/csghub/embedded/etc/postgresql/postgresql.conf.sample similarity index 89% rename from opt/csghub/etc/postgresql/postgresql.conf.sample rename to opt/csghub/embedded/etc/postgresql/postgresql.conf.sample index 9c0fe609..6c541c03 100755 --- a/opt/csghub/etc/postgresql/postgresql.conf.sample +++ b/opt/csghub/embedded/etc/postgresql/postgresql.conf.sample @@ -31,13 +31,10 @@ # TB = terabytes h = hours # d = days -{{- $postgresql := (datasource "config").postgresql }} -{{- $cpuCores := (datasource "resources").cpu_cores }} -{{- $memory := (datasource "resources").memory_mb }} -{{- $disk := (datasource "resources").disk_gb }} -{{- $sharedBuffers := $postgresql.shared_buffers | default "128MB" }} -{{- $sharedBuffersWithoutUnit := regexp.Find "^\\d+" ($sharedBuffers | strings.ToUpper) }} -{{- $memoryAllocated := math.Mul (math.Div $sharedBuffersWithoutUnit 1024) 4 }} +{{- $cpuCores := .resources.cpu_cores }} +{{- $sharedBuffers := .postgresql.shared_buffers | default "128MB" }} +{{- $sharedBuffersWithoutUnit := regexFind "^\\d+" ($sharedBuffers | upper) }} +{{- $memoryAllocated := mulf (div $sharedBuffersWithoutUnit 1024) 4 }} #------------------------------------------------------------------------------ # FILE LOCATIONS @@ -57,20 +54,19 @@ #external_pid_file = '' # write an extra PID file # (change requires restart) - #------------------------------------------------------------------------------ # CONNECTIONS AND AUTHENTICATION #------------------------------------------------------------------------------ # - Connection Settings - -listen_addresses = {{ $postgresql.listen_addresses | default "*" | squote }} +listen_addresses = {{ .postgresql.listen_addresses | default "*" | squote }} # comma-separated list of addresses; # defaults to 'localhost'; use '*' for all # (change requires restart) -port = {{ $postgresql.port | default 5432 }} # (change requires restart) -{{ $maxConnections := math.Min (math.Floor (math.Div $memoryAllocated 20)) 100 }} -max_connections = {{ $postgresql.max_connections | default $maxConnections }} # (change requires restart) +port = {{ .postgresql.port | default 5432 }} # (change requires restart) +{{- $maxConnections := min (floor (div $memoryAllocated 20)) 100 }} +max_connections = {{ .postgresql.max_connections | default $maxConnections }} # (change requires restart) superuser_reserved_connections = 5 # (change requires restart) #unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories # (change requires restart) @@ -125,7 +121,6 @@ superuser_reserved_connections = 5 # (change requires restart) #ssl_passphrase_command = '' #ssl_passphrase_command_supports_reload = off - #------------------------------------------------------------------------------ # RESOURCE USAGE (except WAL) #------------------------------------------------------------------------------ @@ -138,13 +133,13 @@ shared_buffers = {{ $sharedBuffers }} # min 128kB #huge_page_size = 0 # zero for system default # (change requires restart) #temp_buffers = 8MB # min 800kB -max_prepared_transactions = {{ math.Floor (math.Mul (math.Div (math.Mul $memoryAllocated 1000) 4) 5) }} # zero disables the feature +max_prepared_transactions = {{ floor (mulf (div (mul $memoryAllocated 1000) 4) 5) }} # zero disables the feature # (change requires restart) # Caution: it is not advisable to set max_prepared_transactions nonzero unless # you actively intend to use prepared transactions. -work_mem = {{ printf "%dMB" (conv.ToInt (math.Max (math.Min (math.Floor (math.Div (math.Mul $memory 1024) 4096)) 64) 16)) }} # min 64kB +work_mem = {{ printf "%dMB" (int (max (min (floor (div (mul .resources.memory_mb 1024) 4096)) 64) 16)) }} # min 64kB #hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem -maintenance_work_mem = {{ printf "%dMB" (conv.ToInt (math.Min 8192 (math.Floor (math.Div (math.Div (math.Mul $memory 1024) 8) (math.Min (math.Max (math.Floor (math.Div $cpuCores 2)) 2) 16))))) }} # min 1MB +maintenance_work_mem = {{ printf "%dMB" (int (min 8192 (floor (div (div (mul .resources.memory_mb 1024) 8) (min (max (floor (div $cpuCores 2)) 2) 16))))) }} # min 1MB #autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem #logical_decoding_work_mem = 64MB # min 64kB #max_stack_depth = 2MB # min 100kB @@ -193,27 +188,26 @@ bgwriter_lru_multiplier = 10 # 0-10.0 multiplier on buffers scanned/round #backend_flush_after = 0 # measured in pages, 0 disables effective_io_concurrency = 1 # 1-1000; 0 disables prefetching #maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching -max_worker_processes = {{ math.Max (math.Floor (math.Div $cpuCores 2)) 2 }} # (change requires restart) -max_parallel_workers_per_gather = {{ math.Min (math.Max (math.Floor (math.Sub $cpuCores 4)) 2) 24 }} # taken from max_parallel_workers -max_parallel_maintenance_workers = {{ math.Min (math.Max (math.Floor (math.Div $cpuCores 2)) 2) 16 }} # taken from max_parallel_workers -max_parallel_workers = {{ math.Max (math.Floor (math.Sub $cpuCores 4)) 2 }} # maximum number of max_worker_processes that +max_worker_processes = {{ max (floor (div $cpuCores 2)) 2 }} # (change requires restart) +max_parallel_workers_per_gather = {{ min (max (floor (sub $cpuCores 4)) 2) 24 }} # taken from max_parallel_workers +max_parallel_maintenance_workers = {{ min (max (floor (div $cpuCores 2)) 2) 16 }} # taken from max_parallel_workers +max_parallel_workers = {{ max (floor (sub $cpuCores 4)) 2 }} # maximum number of max_worker_processes that # can be used in parallel operations #parallel_leader_participation = on #old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate # (change requires restart) - #------------------------------------------------------------------------------ # WRITE-AHEAD LOG #------------------------------------------------------------------------------ # - Settings - -{{- if has (datasource "config") "patroni" }} -{{- if (datasource "config").patroni.enable }} +{{- if has . "patroni" }} +{{- if .patroni.enable }} wal_level = replica # minimal, replica, or logical {{- end }} {{- else }} -wal_level = {{ $postgresql.wal_level | default "minimal" }} # minimal, replica, or logical +wal_level = {{ .postgresql.wal_level | default "minimal" }} # minimal, replica, or logical {{- end }} # (change requires restart) #fsync = on # flush data to disk for crash safety @@ -229,13 +223,13 @@ wal_level = {{ $postgresql.wal_level | default "minimal" }} # minimal, replica, # fsync_writethrough # open_sync #full_page_writes = on # recover from partial page writes -{{- if has (datasource "config") "patroni" }} -{{- if (datasource "config").patroni.enable }} +{{- if has . "patroni" }} +{{- if .patroni.enable }} wal_log_hints = on {{- else }} {{- end }} {{- else }} -wal_log_hints = {{ $postgresql.wal_log_hints | default "on" }} # also do full page writes of non-critical updates +wal_log_hints = {{ .postgresql.wal_log_hints | default "on" }} # also do full page writes of non-critical updates # (change requires restart) {{- end }} #wal_compression = off # enable compression of full-page writes @@ -252,7 +246,7 @@ wal_log_hints = {{ $postgresql.wal_log_hints | default "on" }} # also do full # - Checkpoints - -checkpoint_timeout = {{ $postgresql.checkpoint_timeout | default "10min" }} # range 30s-1d +checkpoint_timeout = {{ .postgresql.checkpoint_timeout | default "10min" }} # range 30s-1d checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 #checkpoint_flush_after = 256kB # measured in pages, 0 disables #checkpoint_warning = 30s # 0 disables @@ -305,7 +299,6 @@ min_wal_size = 80MB #recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' # (change requires restart) - #------------------------------------------------------------------------------ # REPLICATION #------------------------------------------------------------------------------ @@ -313,13 +306,13 @@ min_wal_size = 80MB # - Sending Servers - # Set these on the primary and on any standby that will send replication data. -{{- if eq $postgresql.wal_level "minimal" }} +{{- if eq .postgresql.wal_level "minimal" }} max_wal_senders = 0 max_replication_slots = 0 {{- else }} -max_wal_senders = {{ $postgresql.max_wal_senders | default 4 }} # max number of walsender processes +max_wal_senders = {{ .postgresql.max_wal_senders | default 4 }} # max number of walsender processes # (change requires restart) -max_replication_slots = {{ $postgresql.max_replication_slots | default 3 }} # max number of replication slots +max_replication_slots = {{ .postgresql.max_replication_slots | default 3 }} # max number of replication slots # (change requires restart) {{- end }} #wal_keep_size = 0 # in megabytes; 0 disables @@ -345,12 +338,12 @@ max_replication_slots = {{ $postgresql.max_replication_slots | default 3 }} # ma #primary_conninfo = '' # connection string to sending server #primary_slot_name = '' # replication slot on sending server #promote_trigger_file = '' # file name whose presence ends recovery -{{- if has (datasource "config") "patroni" }} -{{- if (datasource "config").patroni.enable }} +{{- if has . "patroni" }} +{{- if .patroni.enable }} hot_standby = on # "off" disallows queries during recovery {{- end }} {{- else }} -hot_standby = {{ $postgresql.hot_standby | default "on" }} +hot_standby = {{ .postgresql.hot_standby | default "on" }} {{- end }} # (change requires restart) #max_standby_archive_delay = 30s # max delay before canceling queries @@ -380,7 +373,6 @@ hot_standby = {{ $postgresql.hot_standby | default "on" }} # (change requires restart) #max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers - #------------------------------------------------------------------------------ # QUERY TUNING #------------------------------------------------------------------------------ @@ -419,7 +411,7 @@ random_page_cost = 1.1 # same scale as above #parallel_tuple_cost = 0.1 # same scale as above #min_parallel_table_scan_size = 8MB #min_parallel_index_scan_size = 512kB -effective_cache_size = {{ printf "%dGB" (conv.ToInt (math.Floor (math.Mul (math.Div $memory 1024) 0.75))) }} +effective_cache_size = {{ printf "%dGB" (int (floor (mulf (div .resources.memory_mb 1024) 0.75))) }} #jit_above_cost = 100000 # perform JIT compilation if available # and query more expensive than this; @@ -452,7 +444,6 @@ effective_cache_size = {{ printf "%dGB" (conv.ToInt (math.Floor (math.Mul (math. #plan_cache_mode = auto # auto, force_generic_plan or # force_custom_plan - #------------------------------------------------------------------------------ # REPORTING AND LOGGING #------------------------------------------------------------------------------ @@ -531,7 +522,7 @@ effective_cache_size = {{ printf "%dGB" (conv.ToInt (math.Floor (math.Mul (math. # fatal # panic (effectively off) -log_min_duration_statement = {{ $postgresql.log_min_duration_statement | default "2s" }} # -1 is disabled, 0 logs all statements +log_min_duration_statement = {{ .postgresql.log_min_duration_statement | default "2s" }} # -1 is disabled, 0 logs all statements # and their durations, > 0 logs only # statements running at least this number # of milliseconds @@ -546,7 +537,6 @@ log_min_duration_statement = {{ $postgresql.log_min_duration_statement | default # log_min_duration_sample to be logged; # 1.0 logs all such statements, 0.0 never logs - #log_transaction_sample_rate = 0.0 # fraction of transactions whose statements # are logged regardless of their duration; 1.0 logs all # statements from all transactions, 0.0 never logs @@ -562,10 +552,10 @@ log_min_duration_statement = {{ $postgresql.log_min_duration_statement | default # their durations, > 0 logs only # actions running at least this number # of milliseconds. -log_checkpoints = {{ $postgresql.log_checkpoints | default "off" }} -log_connections = {{ $postgresql.log_connections | default "off" }} -log_disconnections = {{ $postgresql.log_disconnections | default "off" }} -log_duration = {{ $postgresql.log_duration | default "off" }} +log_checkpoints = {{ .postgresql.log_checkpoints | default "off" }} +log_connections = {{ .postgresql.log_connections | default "off" }} +log_disconnections = {{ .postgresql.log_disconnections | default "off" }} +log_duration = {{ .postgresql.log_duration | default "off" }} #log_error_verbosity = default # terse, default, or verbose messages #log_hostname = off #log_line_prefix = '%m [%p] ' # special values: @@ -601,13 +591,12 @@ log_lock_waits = on # log lock waits >= deadlock_timeout #log_parameter_max_length_on_error = 0 # when logging an error, limit logged # bind-parameter values to N bytes; # -1 means print in full, 0 disables -log_statement = {{ $postgresql.log_statement | default "none" }} # none, ddl, mod, all +log_statement = {{ .postgresql.log_statement | default "none" }} # none, ddl, mod, all #log_replication_commands = off #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; # -1 disables, 0 logs all temp files -log_timezone = {{ $postgresql.timezone | default "Asia/Shanghai" | squote }} - +log_timezone = {{ .postgresql.timezone | default "Asia/Shanghai" | squote }} #------------------------------------------------------------------------------ # PROCESS TITLE @@ -617,7 +606,6 @@ log_timezone = {{ $postgresql.timezone | default "Asia/Shanghai" | squote }} # (change requires restart) #update_process_title = on - #------------------------------------------------------------------------------ # STATISTICS #------------------------------------------------------------------------------ @@ -632,7 +620,6 @@ log_timezone = {{ $postgresql.timezone | default "Asia/Shanghai" | squote }} #track_functions = none # none, pl, all #stats_temp_directory = 'pg_stat_tmp' - # - Monitoring - #compute_query_id = auto @@ -641,14 +628,13 @@ log_timezone = {{ $postgresql.timezone | default "Asia/Shanghai" | squote }} #log_planner_stats = off #log_executor_stats = off - #------------------------------------------------------------------------------ # AUTOVACUUM #------------------------------------------------------------------------------ #autovacuum = on # Enable autovacuum subprocess? 'on' # requires track_counts to also be on. -autovacuum_max_workers = {{ math.Max (math.Min (math.Floor (math.Div $cpuCores 2)) 8) 5 }} # max number of autovacuum subprocesses +autovacuum_max_workers = {{ max (min (floor (div $cpuCores 2)) 8) 5 }} # max number of autovacuum subprocesses # (change requires restart) #autovacuum_naptime = 1min # time between autovacuum runs #autovacuum_vacuum_threshold = 50 # min number of row updates before @@ -674,7 +660,6 @@ autovacuum_analyze_scale_factor = 0.01 # fraction of table size before analyze # autovacuum, -1 means use # vacuum_cost_limit - #------------------------------------------------------------------------------ # CLIENT CONNECTION DEFAULTS #------------------------------------------------------------------------------ @@ -722,7 +707,7 @@ idle_in_transaction_session_timeout = 60000 # in milliseconds, 0 is disabled datestyle = 'iso, mdy' #intervalstyle = 'postgres' -timezone = {{ $postgresql.timezone | default "Asia/Shanghai" | squote }} +timezone = {{ .postgresql.timezone | default "Asia/Shanghai" | squote }} #timezone_abbreviations = 'Default' # Select the set of available time zone # abbreviations. Currently, there are # Default @@ -749,7 +734,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = {{ $postgresql.shared_preload_libraries | default "pg_stat_statements" | squote }} # (change requires restart) +shared_preload_libraries = {{ .postgresql.shared_preload_libraries | default "pg_stat_statements" | squote }} # (change requires restart) #jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - @@ -759,7 +744,6 @@ shared_preload_libraries = {{ $postgresql.shared_preload_libraries | default "pg # and shared objects (added by Debian) #gin_fuzzy_search_limit = 0 - #------------------------------------------------------------------------------ # LOCK MANAGEMENT #------------------------------------------------------------------------------ @@ -774,7 +758,6 @@ max_locks_per_transaction = 128 # min 10 # / -max_pred_locks_per_relation) - 1 #max_pred_locks_per_page = 2 # min 0 - #------------------------------------------------------------------------------ # VERSION AND PLATFORM COMPATIBILITY #------------------------------------------------------------------------------ @@ -793,7 +776,6 @@ max_locks_per_transaction = 128 # min 10 #transform_null_equals = off - #------------------------------------------------------------------------------ # ERROR HANDLING #------------------------------------------------------------------------------ @@ -805,7 +787,6 @@ max_locks_per_transaction = 128 # min 10 # (change requires restart) #recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) - #------------------------------------------------------------------------------ # CONFIG FILE INCLUDES #------------------------------------------------------------------------------ @@ -819,10 +800,9 @@ max_locks_per_transaction = 128 # min 10 #include_if_exists = '...' # include file only if it exists #include = '...' # include file - #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS #------------------------------------------------------------------------------ # Add settings for extensions here -pg_stat_statements.track = all \ No newline at end of file +pg_stat_statements.track = all diff --git a/opt/csghub/etc/praefect/config.toml.sample b/opt/csghub/embedded/etc/praefect/config.toml.sample similarity index 56% rename from opt/csghub/etc/praefect/config.toml.sample rename to opt/csghub/embedded/etc/praefect/config.toml.sample index 36a2c8ed..4c564e80 100755 --- a/opt/csghub/etc/praefect/config.toml.sample +++ b/opt/csghub/embedded/etc/praefect/config.toml.sample @@ -4,84 +4,82 @@ # and run: # sudo csghub-ctl reconfigure -{{- $praefect := (datasource "config").praefect }} # TCP address to listen on -listen_addr = {{ $praefect.listen_addr | quote }} +listen_addr = {{ .praefect.listen_addr | quote }} -{{- if $praefect.tls_listen_addr }} +{{- if .praefect.tls_listen_addr }} # Secured TCP address to listen on. -tls_listen_addr = {{ $praefect.tls_listen_addr | quote }} +tls_listen_addr = {{ .praefect.tls_listen_addr | quote }} {{- end }} - -{{- if has $praefect "tls" }} +{{- if has .praefect "tls" }} # Path to the certificate and its key used for TLS listening address. [tls] -{{- if $praefect.tls.certificate_path }} -certificate_path = {{ $praefect.tls.certificate_path | quote }} +{{- if .praefect.tls.certificate_path }} +certificate_path = {{ .praefect.tls.certificate_path | quote }} {{- end }} -{{- if $praefect.tls.key_path }} -key_path = {{ $praefect.tls.key_path | quote }} +{{- if .praefect.tls.key_path }} +key_path = {{ .praefect.tls.key_path | quote }} {{- end }} {{- end }} # Optional: grace period before a praefect process is forcibly terminated (duration) # Defaults to "1m" -graceful_stop_timeout = {{ $praefect.graceful_stop_timeout | quote }} +graceful_stop_timeout = {{ .praefect.graceful_stop_timeout | quote }} # Optional: export metrics via Prometheus -prometheus_listen_addr = {{ $praefect.prometheus_listen_addr | quote }} +prometheus_listen_addr = {{ .praefect.prometheus_listen_addr | quote }} # You can optionally configure Praefect to output JSON-formatted log messages to stdout [logging] -format = {{ $praefect.logging.format | quote }} +format = {{ .praefect.logging.format | quote }} # Optional: Set log level to only log entries with that severity or above # One of, in order: debug, info, warn, error, fatal, panic # Defaults to "info" -level = {{ $praefect.logging.level | quote }} +level = {{ .praefect.logging.level | quote }} [prometheus] -grpc_latency_buckets = [{{- range $i, $v := $praefect.prometheus.grpc_latency_buckets }}{{if $i}}, {{end}}{{$v}}{{- end }}] +grpc_latency_buckets = [{{- range $i, $v := .praefect.prometheus.grpc_latency_buckets }}{{if $i}}, {{end}}{{$v}}{{- end }}] [sentry] -sentry_environment = {{ $praefect.sentry.sentry_environment | quote }} -sentry_dsn = {{ $praefect.sentry.sentry_dsn | quote }} +sentry_environment = {{ .praefect.sentry.sentry_environment | default "" | quote }} +sentry_dsn = {{ .praefect.sentry.sentry_dsn | default "" | quote }} -{{ $pgUser := $praefect.database.user }} -{{- $pgPassword := $praefect.database.password | default (crypto.PBKDF2 $pgUser "opencsg" 2048 8) }} +{{- $pgUser := .praefect.database.user }} +{{- $pgPassword := .praefect.database.password | default (pbkdf2 $pgUser "opencsg" 2048 8) }} [database] -host = {{ $praefect.database.host | quote }} -port = {{ $praefect.database.port }} +host = {{ .praefect.database.host | quote }} +port = {{ .praefect.database.port }} user = {{ $pgUser | quote }} password = {{ $pgPassword | quote }} -dbname = {{ $praefect.database.dbname | quote }} +dbname = {{ .praefect.database.dbname | quote }} # Optional: authenticate Gitaly requests using a shared secret. This token works the same way as a gitaly token [auth] -token = {{ $praefect.auth.token | default (crypto.PBKDF2 "praefect" "opencsg" 2048 8) | quote }} +token = {{ .praefect.auth.token | default (pbkdf2 "praefect" "opencsg" 2048 8) | quote }} # One or more Gitaly servers need to be configured to be managed. The names # of each server are used to link multiple nodes, or `gitaly_server`s together # as shard. listen_addr should be unique for all nodes. # Requires the protocol to be defined, e.g. tcp://host.tld:1234 [replication] -batch_size = {{ $praefect.replication.batch_size }} +batch_size = {{ .praefect.replication.batch_size }} [reconciliation] # Duration value specifying an interval at which to run the automatic repository reconciler. # Automatic reconciliation is disabled if set to 0. Example: "1m" for reconciliation every minute. -scheduling_interval = "{{ $praefect.reconciliation.scheduling_interval }}" +scheduling_interval = "{{ .praefect.reconciliation.scheduling_interval }}" # Scheduling duration histogram buckets. -histogram_buckets = [{{- range $i, $v := $praefect.reconciliation.histogram_buckets }}{{if $i}}, {{end}}{{$v}}{{- end }}] +histogram_buckets = [{{- range $i, $v := .praefect.reconciliation.histogram_buckets }}{{if $i}}, {{end}}{{$v}}{{- end }}] [failover] -enabled = {{ $praefect.failover.enabled }} +enabled = {{ .praefect.failover.enabled }} [background_verification] -delete_invalid_records = {{ $praefect.background_verification.delete_invalid_records }} -verification_interval = {{ $praefect.background_verification.verification_interval | quote }} +delete_invalid_records = {{ .praefect.background_verification.delete_invalid_records }} +verification_interval = {{ .praefect.background_verification.verification_interval | quote }} -{{ if has $praefect "virtual_storage" }} -{{- range $vs := $praefect.virtual_storage }} +{{- if has .praefect "virtual_storage" }} +{{- range $vs := .praefect.virtual_storage }} [[virtual_storage]] name = "{{ $vs.name }}" {{- if $vs.default_replication_factor }} @@ -91,17 +89,16 @@ default_replication_factor = {{ $vs.default_replication_factor }} [[virtual_storage.node]] storage = "{{ $node.storage }}" address = "{{ $node.address }}" -token = "{{ $node.token | default (crypto.PBKDF2 "gitaly" "opencsg" 2048 8) }}" +token = "{{ $node.token | default (pbkdf2 "gitaly" "opencsg" 2048 8) }}" {{- end }} {{- end }} {{- end }} - -{{ if has $praefect "yamux" }} +{{- if has .praefect "yamux" }} [yamux] # MaximumStreamWindowSizeBytes sets the maximum window size in bytes used for yamux streams. # Higher value can increase throughput at the cost of more memory usage. -maximum_stream_window_size_bytes = {{ $praefect.yamux.maximum_stream_window_size_bytes | default 262144 }} +maximum_stream_window_size_bytes = {{ .praefect.yamux.maximum_stream_window_size_bytes | default 262144 }} # AcceptBacklog sets the maximum number of stream openings in-flight # before further openings block. -accept_backlog = {{ $praefect.yamux.accept_backlog | default 256 }} -{{ end }} +accept_backlog = {{ .praefect.yamux.accept_backlog | default 256 }} +{{- end }} diff --git a/opt/csghub/embedded/etc/prometheus/prometheus.yml.sample b/opt/csghub/embedded/etc/prometheus/prometheus.yml.sample new file mode 100644 index 00000000..2a879378 --- /dev/null +++ b/opt/csghub/embedded/etc/prometheus/prometheus.yml.sample @@ -0,0 +1,22 @@ +global: + scrape_interval: {{ .prometheus.scrape_interval | default "15s" }} + evaluation_interval: {{ .prometheus.evaluation_interval | default "15s" }} + scrape_timeout: {{ .prometheus.scrape_timeout | default "10s" }} + +{{- if .prometheus.external_labels }} + external_labels: + {{- range $key, $value := .prometheus.external_labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} +{{- end }} + +rule_files: + # - "alert.rules.yml" + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] +{{- if .prometheus.scrape_configs }} +{{ toYaml .prometheus.scrape_configs | indent 2 }} +{{- end }} diff --git a/opt/csghub/etc/redis/redis.conf.sample b/opt/csghub/embedded/etc/redis/redis.conf.sample similarity index 99% rename from opt/csghub/etc/redis/redis.conf.sample rename to opt/csghub/embedded/etc/redis/redis.conf.sample index 372979fc..723770d2 100644 --- a/opt/csghub/etc/redis/redis.conf.sample +++ b/opt/csghub/embedded/etc/redis/redis.conf.sample @@ -17,8 +17,7 @@ # # units are case insensitive so 1GB 1Gb 1gB are all the same. -{{- $redis := (datasource "config").redis }} -{{- $dataDir := $redis.data | default "/var/opt/csghub/redis" }} +{{- $dataDir := .redis.data | default "/var/opt/csghub/redis" }} ################################## INCLUDES ################################### @@ -75,8 +74,8 @@ # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES # JUST COMMENT OUT THE FOLLOWING LINE. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -{{- if $redis.listen }} -bind {{ $redis.listen | default "127.0.0.1" }} +{{- if .redis.listen }} +bind {{ .redis.listen | default "127.0.0.1" }} {{- else }} bind * -::* {{- end }} @@ -102,7 +101,7 @@ protected-mode yes # Accept connections on the specified port, default is 6379 (IANA #815344). # If port 0 is specified Redis will not listen on a TCP socket. -port {{ $redis.port | default 6379 }} +port {{ .redis.port | default 6379 }} # TCP listen() backlog. # @@ -123,7 +122,7 @@ tcp-backlog 511 # unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) -timeout {{ $redis.timeout | default 0 }} +timeout {{ .redis.timeout | default 0 }} # TCP keepalive. # @@ -140,7 +139,7 @@ timeout {{ $redis.timeout | default 0 }} # # A reasonable value for this option is 300 seconds, which is the new # Redis default starting with Redis 3.2.1. -tcp-keepalive {{ $redis.keepalive | default 300 }} +tcp-keepalive {{ .redis.keepalive | default 300 }} ################################# TLS/SSL ##################################### @@ -303,8 +302,8 @@ pidfile {{ printf "%s/redis_6379.pid" $dataDir }} # verbose (many rarely useful info, but not a mess like the debug level) # notice (moderately verbose, what you want in production probably) # warning (only very important / critical messages are logged) -{{- if has $redis "log" }} -loglevel {{ $redis.log.level }} +{{- if has .redis "log" }} +loglevel {{ .redis.log.level }} {{- else }} loglevel notice {{- end }} @@ -911,7 +910,7 @@ acllog-max-len 128 # The requirepass is not compatable with aclfile option and the ACL LOAD # command, these will cause requirepass to be ignored. # -# requirepass {{ $redis.password | default (crypto.PBKDF2 $redis.user "opencsg" 2048 8) }} +# requirepass {{ .redis.password | default (pbkdf2 .redis.user "opencsg" 2048 8) }} # New users are initialized with restrictive permissions by default, via the # equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it @@ -1229,7 +1228,6 @@ oom-score-adj no # oom-score-adj-values to positive values will always succeed. oom-score-adj-values 0 200 800 - #################### KERNEL transparent hugepage CONTROL ###################### # Usually the kernel Transparent Huge Pages control is set to "madvise" or diff --git a/opt/csghub/etc/registry/config.yml.sample b/opt/csghub/embedded/etc/registry/config.yml.sample similarity index 66% rename from opt/csghub/etc/registry/config.yml.sample rename to opt/csghub/embedded/etc/registry/config.yml.sample index 42264f39..1f4c0e0f 100644 --- a/opt/csghub/etc/registry/config.yml.sample +++ b/opt/csghub/embedded/etc/registry/config.yml.sample @@ -1,16 +1,14 @@ -{{- $registry := (datasource "config").registry -}} -{{- $dataDir := $registry.data -}} -{{- $minio := (datasource "config").minio -}} -{{- $user := $minio.auth.user -}} -{{- $password := $minio.auth.password | default (crypto.PBKDF2 $user "opencsg" 2048 8) -}} -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} +{{- $dataDir := .registry.data }} +{{- $user := .minio.auth.user }} +{{- $password := .minio.auth.password | default (pbkdf2 $user "opencsg" 2048 8) }} +{{- $csghub := resolveUrl .csghub.external_url }} version: 0.1 log: - {{- $logLevel := "info" -}} + {{- $logLevel := "info" }} {{- $logFormatter := "text" }} - {{- if has $registry "log" -}} - {{- $logLevel = $registry.log.level -}} - {{- $logFormatter = $registry.log.formatter }} + {{- if has .registry "log" }} + {{- $logLevel = .registry.log.level }} + {{- $logFormatter = .registry.log.formatter }} {{- end }} level: {{ $logLevel }} formatter: {{ $logFormatter }} @@ -19,13 +17,13 @@ log: storage: cache: blobdescriptor: inmemory - {{- if has $registry.storage "filesystem" -}} + {{- if has .registry.storage "filesystem" }} filesystem: - rootdirectory: {{ $registry.storage.filesystem.rootdirectory }} - maxthreads: {{ $registry.storage.filesystem.maxthreads }} + rootdirectory: {{ .registry.storage.filesystem.rootdirectory }} + maxthreads: {{ .registry.storage.filesystem.maxthreads }} {{- else }} - {{- if has $registry.storage "s3" }} - {{- $auth := $registry.storage.s3 }} + {{- if has .registry.storage "s3" }} + {{- $auth := .registry.storage.s3 }} s3: accesskey: {{ $auth.access_key | default $user }} secretkey: {{ $auth.secret_key | default $password }} @@ -38,8 +36,8 @@ storage: v4auth: {{ $auth.v4auth | default true }} skipverify: {{ $auth.skip_verify | default true }} {{- end }} - {{- if has $registry.storage "oss" }} - {{- $auth := $registry.storage.oss }} + {{- if has .registry.storage "oss" }} + {{- $auth := .registry.storage.oss }} oss: accesskeyid: {{ $auth.access_key_id | default $user }} accesskeysecret: {{ $auth.access_key_secret | default $password }} diff --git a/opt/csghub/embedded/etc/server/config.toml.sample b/opt/csghub/embedded/etc/server/config.toml.sample new file mode 100644 index 00000000..4dab7b1d --- /dev/null +++ b/opt/csghub/embedded/etc/server/config.toml.sample @@ -0,0 +1,308 @@ +{{- $csghub := resolveUrl .csghub.external_url }} +saas = {{ .server.saas }} +oversea = false +instance_id = "" +enable_swagger = {{ .server.enable_swagger }} +enable_https = {{ .server.enable_https }} +api_token = {{ deriveToken | quote }} +docs_host = {{ printf "%s://%s:6636" $csghub.scheme $csghub.host | quote }} +server_failure_redirect_url = {{ printf "%s/errors/server-error" $csghub.url | quote }} + +[api_server] +port = {{ .server.listen_port }} +public_domain = {{ $csghub.url | quote }} +{{- $shell_port := .gitlab_shell.ssh_port }} +{{- if eq $shell_port 22 }} +ssh_domain = {{ printf "git@%s:%d" (toString $csghub.host) (int64 $shell_port) | quote }} +{{- else }} +ssh_domain = {{ printf "ssh://git@%s:%d" (toString $csghub.host) (int64 $shell_port) | quote }} +{{- end }} + +[mirror] +url = {{ printf "http://localhost:%d" .mirror_repo.listen_port | quote }} +token = "" +port = {{ .mirror_repo.listen_port }} +{{- if .server.saas }} +remote = true +{{- else }} +remote = false +{{- end }} +session_secret_key = {{ persistentSeed | sha1sum | quote }} +worker_number = {{ .mirror_repo.worker_number }} +lfs_concurrency = {{ .mirror_lfs.lfs_concurrency }} +part_size = {{ .mirror_lfs.part_size }} + +{{- $postgresql := .server.postgresql }} +{{- $pgUser := $postgresql.user }} +{{- $pgPassword := $postgresql.password | default (pbkdf2 $pgUser "opencsg" 2048 8) }} + +[database] +driver = "pg" +dsn = {{ printf "postgresql://%s:%s@%s:%d/%s?sslmode=disable" (toString $pgUser) (toString $pgPassword) (toString $postgresql.host) (int64 $postgresql.port) (toString $postgresql.name) | quote }} +timezone = {{ $postgresql.timezone | quote }} + +{{- $redis := .server.redis }} +{{- $redisUser := $redis.auth.user }} +{{- $redisPassword := $redis.auth.password | default (pbkdf2 $redisUser "opencsg" 2048 8) }} + +[redis] +endpoint = {{ $redis.endpoint | quote }} +max_retries = {{ $redis.max_retries }} +min_idle_connections = {{ $redis.min_idle_connections }} +user = {{ $redisUser | quote }} +password = {{ $redisPassword | quote }} +sentinel_mode = {{ $redis.sentinel.mode }} +sentinel_endpoint = {{ $redis.sentinel.endpoint | quote }} +sentinel_master = {{ $redis.sentinel.master | quote }} + +[git_server] +type = "gitaly" + +{{- $gitaly := .server.gitaly }} +{{- $gitalyToken := $gitaly.token | default (pbkdf2 "gitaly" "opencsg" 2048 8) }} + +[gitaly_server] +address = {{ $gitaly.address | quote }} +storge = {{ $gitaly.storage | quote }} +token = {{ $gitalyToken | quote }} +jwt_secret ={{ $gitaly.secret | default (persistentSeed | sha256sum) | quote }} + +[frontend] +url = {{ $csghub.url | quote }} + +{{- $s3 := .server.s3 }} +{{- $accessKeyId := $s3.access_key }} +{{- $secretKey := $s3.secret_key | default (pbkdf2 $accessKeyId "opencsg" 2048 8) }} + +[s3] +access_key_id = {{ $accessKeyId | quote }} +access_key_secret = {{ $secretKey | quote }} +region = {{ $s3.region | quote }} +endpoint = {{ $s3.endpoint | default (printf "%s:9000" $csghub.host) | quote }} +internal_endpoint = {{ $s3.internal_endpoint | quote }} +bucket = {{ $s3.bucket | quote }} +enable_ssl = {{ $s3.secure }} +{{- if $s3.path_style }} +bucket_lookup = "path" +{{- else }} +bucket_lookup = "auto" +{{- end }} + +[jwt] +signing_key = {{ $gitaly.secret | default (persistentSeed | sha256sum) | quote }} +valid_hour = 24 + +[space] +internal_root_domain = "spaces.app.internal:8083" +{{- if .runner.use_public_domain }} +public_root_domain = {{ printf "%s:%v" $csghub.host $csghub.port | quote }} +{{- else }} +public_root_domain = "" +{{- end }} +session_secret_key = {{ persistentSeed | sha1sum | quote }} +pypi_index_url = {{ .runner.pip_index_url | quote }} + +{{- $model := .runner.model }} + +[model] +deploy_timeout_in_min = {{ $model.deploy_timeout }} +download_endpoint = {{ $csghub.url | quote }} +docker_reg_base = {{ $model.registry | quote }} +nim_docker_secret_name = "ngc-secret" +nim_ngc_secret_name = "nvidia-nim-secrets" + +[event] +sync_interval = {{ .mirror_repo.sync_interval }} + +{{- $serverCasdoor := .server.casdoor }} +{{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} + +[casdoor] +client_id = {{ generateClientId "CSGHub" | quote }} +client_secret = {{ generateClientSecret "CSGHub" | quote }} +endpoint = {{ $serverCasdoor.endpoint | default ($casdoorEp.url) | quote }} +certificate = {{ $serverCasdoor.certificate | quote }} +organization_name = {{ $serverCasdoor.organization_name | quote }} +application_name = {{ $serverCasdoor.application_name | quote }} + +{{- $natsUser := .nats.auth.user }} +{{- $natsPassword := .nats.auth.password | default (pbkdf2 $natsUser "opencsg" 2048 8) }} +{{- $serverNats := .server.nats }} + +[nats] +url = {{ $serverNats.url | default (printf "nats://%s:%s@localhost:4222" (toString $natsUser) (toString $natsPassword)) | quote }} +msg_fetch_timeout_in_sec = {{ $serverNats.msg_fetch_timeout_in_sec }} + +{{- $actParts := .accounting.listen | toString | splitList ":" }} +{{- $actHost := index $actParts 0 }} +{{- $actPort := "" }} +{{- if gt (len $actParts) 1 }} + {{- $actPort = index $actParts 1 }} +{{- else }} + {{- $actPort = "8086" }} +{{- end }} + +[accounting] +host = {{ printf "http://%s" $actHost | quote }} +port = {{ $actPort }} + +{{- $userParts := .user.listen | splitList ":" }} +{{- $userHost := index $userParts 0 }} +{{- $userPort := "" }} +{{- if gt (len $userParts) 1 }} + {{- $userPort = index $userParts 1 }} +{{- else }} + {{- $actPort = "8088" }} +{{- end }} + +[user] +host = {{ printf "http://%s" $userHost | quote }} +port = {{ $userPort }} +signin_success_redirect_url = {{ printf "%s/server/callback" $csghub.url | quote }} + +{{- $multi_sync := .server.multi_sync }} + +[multi_sync] +enabled = {{ $multi_sync.enabled }} +saas_api_domain = "https://hub.opencsg.com" +saas_sync_domain = "https://sync.opencsg.com" + +{{- $telemetry := .server.telemetry }} + +[telemetry] +enable = {{ $telemetry.enabled | default true }} +report_url = "http://hub.opencsg.com/api/v1/telemetry" + +[auto_clean] +instance = false + +{{- $dataset := .server.dataset }} + +[dataset] +prompt_max_jsonl_file_size = {{ $dataset.prompt_max_jsonl_file_size }} + +{{- $dataflow := .server.dataflow }} +{{- $dfUrl := parseURL $dataflow.address }} +{{- $dfUrlParts := $dfUrl.Host | splitList ":" }} +{{- $dfHost := index $dfUrlParts 0 }} +{{- $dfPort := "" }} +{{- if gt (len $dfUrlParts) 1 }} + {{- $dfPort = index $dfUrlParts 1 }} +{{- else }} + {{- if eq $dfUrl.Scheme "https" }} + {{- $dfPort = "443" }} + {{- else }} + {{- $dfPort = "80" }} + {{- end }} +{{- end }} + +[dataflow] +host = {{ printf "%s://%s" $dfUrl.Scheme $dfHost | quote }} +port = {{ $dfPort }} + +{{- if .moderation.enable }} +{{- $moderationAddress := .moderation.address }} +{{- $moderationUrl := parseURL $moderationAddress }} +{{- $moderationParts := $moderationUrl.Host | splitList ":" }} +{{- $moderationHost := index $moderationParts 0 | default "127.0.0.1" }} +{{- $moderationPort := "" }} +{{- if gt (len $moderationParts) 1 }} + {{- $moderationPort = index $moderationParts 1 }} +{{- else }} + {{- $moderationPort = "8089" }} +{{- end }} + +[moderation] +host = {{ printf "%s://%s" $moderationUrl.Scheme $moderationHost | quote }} +port = {{ $moderationPort }} +encoded_sensitive_words = {{ .moderation.encoded_sensitive_words | quote }} + +{{- $sensitive := .moderation.sensitive_check }} + +[sensitive_check] +enable = true +access_key_id = {{ $sensitive.access_key_id | quote }} +access_key_secret = {{ $sensitive.access_key_secret | quote }} +region = {{ $sensitive.region | quote }} +endpoint = {{ $sensitive.endpoint | quote }} +enable_ssl = {{ $sensitive.enable_ssl }} +{{- end }} +{{- $workflow := .server.workflow }} + +[workflow] +endpoint = {{ $workflow.endpoint | quote }} + +[cron_job] +sync_as_client_cron_expression = {{ $multi_sync.sync_as_client_cron_expression | quote }} +calc_recom_score_cron_expression = {{ $multi_sync.calc_recom_score_cron_expression | quote }} + +{{- $proxy := .server.proxy }} + +[proxy] +hosts = [{{- range $k, $v := $proxy.hosts }}{{if $k}}, {{end}}{{$v | quote }}{{- end }}] + +{{- $instrumentation := .server.instrumentation }} + +[instrumentation] +otlp_logging = {{ $instrumentation.otlp_logging }} +otlp_endpoint = {{ $instrumentation.otlp_endpoint | quote }} + +{{- $git := .server.git }} + +[git] +operation_timeout = {{ $git.operation_timeout }} +{{- if .praefect.enable }} +check_file_size_enabled = false +{{- else }} +check_file_size_enabled = {{ $git.check_file_size_enabled }} +{{- end }} +max_un_lfs_file_size = {{ $git.max_un_lfs_file_size }} +skip_lfs_file_validation = {{ $git.skip_lfs_file_validation }} + +{{- if .notifier.enable }} +{{- $notifyListen := .notifier.listen }} +{{- $notifyParts := $notifyListen | splitList ":" }} +{{- $notifyHost := index $notifyParts 0 | default "127.0.0.1" }} +{{- $notifyPort := "8095" }} +{{- if eq (len $notifyParts) 2 }} + {{- $notifyPort = index $notifyParts 1 }} +{{- end }} + +[notification] +host = {{ printf "http://%s" $notifyHost | quote }} +port = {{ $notifyPort }} +mailer_host = {{ .notifier.smtp.host | quote }} +mailer_port = {{ .notifier.smtp.port }} +mailer_username = {{ .notifier.smtp.username | quote }} +mailer_password = {{ .notifier.smtp.password | quote }} +repo_sync_timezone = {{ .notifier.repo_sync_timezone | quote }} +broadcast_user_page_size = {{ .notifier.broadcast_user_page_size }} +broadcast_email_page_size = {{ .notifier.broadcast_email_page_size }} + +{{- $feishu := .notifier.feishu }} + +[feishu] +app_id = {{ $feishu.app_id | quote }} +app_secret = {{ $feishu.app_secret | quote }} +batch_send_message_cron_expression = {{ $feishu.batch_send_message_cron_expression | quote }} +max_request_content_size = {{ $feishu.max_request_content_size }} +max_delay_duration = {{ $feishu.max_delay_duration }} +chat_ids_cache_ttl = {{ $feishu.chat_ids_cache_ttl }} +{{- end }} +{{- $prometheus := .server.prometheus }} + +[prometheus] +api_address = {{ printf "%s/api/v1/query" $prometheus.address | quote }} + +{{- $logcollector := .server.logcollector }} + +[logcollector] +loki_url = {{ $logcollector.loki_url | quote }} + +[aigateway] +port = {{ .aigateway.port }} +moderation_bypass_sensitive_check = {{ .aigateway.moderation_bypass_sensitive_check }} +advertise_addr = "{{ $csghub.host }}:8094" + +[storagegateway] +enable_presigned_url_proxy = {{ .server.enable_presigned_url_proxy }} diff --git a/opt/csghub/etc/temporal/auto-setup.sh b/opt/csghub/embedded/etc/temporal/auto-setup.sh similarity index 100% rename from opt/csghub/etc/temporal/auto-setup.sh rename to opt/csghub/embedded/etc/temporal/auto-setup.sh diff --git a/opt/csghub/etc/temporal/docker.yaml.sample b/opt/csghub/embedded/etc/temporal/docker.yaml.sample similarity index 84% rename from opt/csghub/etc/temporal/docker.yaml.sample rename to opt/csghub/embedded/etc/temporal/docker.yaml.sample index fc9730b3..51193418 100644 --- a/opt/csghub/etc/temporal/docker.yaml.sample +++ b/opt/csghub/embedded/etc/temporal/docker.yaml.sample @@ -1,23 +1,22 @@ -{{- $temporal := (datasource "config").temporal -}} log: stdout: true - level: {{ $temporal.log.level }} + level: {{ .temporal.log.level }} persistence: - numHistoryShards: {{ $temporal.persistence.num_history_shards }} - defaultStore: {{ $temporal.persistence.default_store }} - visibilityStore: {{ $temporal.persistence.visibility_store }} + numHistoryShards: {{ .temporal.persistence.num_history_shards }} + defaultStore: {{ .temporal.persistence.default_store }} + visibilityStore: {{ .temporal.persistence.visibility_store }} datastores: - {{- $sql := $temporal.persistence.datastores.default.sql }} + {{- $sql := .temporal.persistence.datastores.default.sql }} default: - {{- $db := $temporal.postgresql }} + {{- $db := .temporal.postgresql }} sql: pluginName: "postgres12" databaseName: {{ $db.name | quote }} connectAddr: {{ printf "%s:%d" $db.host $db.port | quote }} connectProtocol: "tcp" user: {{ $db.user | quote }} - password: {{ $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) | quote }} + password: {{ $db.password | default (pbkdf2 $db.user "opencsg" 2048 8) | quote }} maxConns: {{ $sql.max_conns }} maxIdleConns: {{ $sql.max_idle_conns }} maxConnLifetime: {{ $sql.max_conn_lifetime }} @@ -30,7 +29,7 @@ persistence: connectAddr: {{ printf "%s:%d" $db.host $db.port | quote }} connectProtocol: "tcp" user: {{ $db.user | quote }} - password: {{ $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) | quote }} + password: {{ $db.password | default (pbkdf2 $db.user "opencsg" 2048 8) | quote }} maxConns: {{ $sql.max_conns }} maxIdleConns: {{ $sql.max_idle_conns }} maxConnLifetime: {{ $sql.max_conn_lifetime }} @@ -84,7 +83,7 @@ global: services: frontend: rpc: - {{- $rpc := $temporal.rpc }} + {{- $rpc := .temporal.rpc }} grpcPort: {{ $rpc.grpc_port }} membershipPort: {{ $rpc.membership_port }} bindOnIP: {{ $rpc.bind_on_ip | quote }} @@ -147,4 +146,4 @@ namespaceDefaults: URI: "file:///tmp/temporal_archival/development" visibility: state: "disabled" - URI: "file:///tmp/temporal_vis_archival/development" \ No newline at end of file + URI: "file:///tmp/temporal_vis_archival/development" diff --git a/opt/csghub/etc/temporal_ui/docker.yaml.sample b/opt/csghub/embedded/etc/temporal_ui/docker.yaml.sample similarity index 55% rename from opt/csghub/etc/temporal_ui/docker.yaml.sample rename to opt/csghub/embedded/etc/temporal_ui/docker.yaml.sample index 3066382f..e909c4e5 100644 --- a/opt/csghub/etc/temporal_ui/docker.yaml.sample +++ b/opt/csghub/embedded/etc/temporal_ui/docker.yaml.sample @@ -1,10 +1,7 @@ -{{- $temporal := (datasource "config").temporal -}} -{{- $temporalUI := (datasource "config").temporal_ui -}} -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML }} -{{- $server := (datasource "config").server }} -temporalGrpcAddress: {{ printf "%s:%d" $temporal.rpc.bind_on_ip $temporal.rpc.grpc_port }} -port: {{ $temporalUI.listen_port }} -publicPath: {{ $temporalUI.public_path }} +{{- $csghub := resolveUrl .csghub.external_url }} +temporalGrpcAddress: {{ printf "%s:%d" .temporal.rpc.bind_on_ip .temporal.rpc.grpc_port }} +port: {{ .temporal_ui.listen_port }} +publicPath: {{ .temporal_ui.public_path }} enableUi: true bannerText: cloudUi: false @@ -23,11 +20,11 @@ batchActionsDisabled: false startWorkflowDisabled: true hideWorkflowQueryErrors: false cors: - cookieInsecure: {{ $temporalUI.cors.cookie_insecure }} + cookieInsecure: {{ .temporal_ui.cors.cookie_insecure }} allowOrigins: # override framework's default that allows all origins "*" - - {{ $csghub.external }} - {{- range $temporalUI.cors.allow_origins }} + - {{ $csghub.url }} + {{- range .temporal_ui.cors.allow_origins }} - {{ . }} {{- end }} @@ -40,7 +37,9 @@ tls: keyData: enableHostVerification: false serverName: -{{- $provider := $server.casdoor.endpoint | default (tmpl.Exec "endpoint.casdoor" .) }} +{{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} +{{- $provider := .server.casdoor.endpoint | default $casdoorEp.url }} + auth: enabled: true providers: @@ -48,13 +47,14 @@ auth: type: oidc providerUrl: {{ $provider }} issuerUrl: {{ $provider }} - clientId: {{ tmpl.Exec "GenClientId" "Admin" }} - clientSecret: {{ tmpl.Exec "GenClientSecret" "Admin" }} - callbackUrl: {{ printf "%s/-/temporal/auth/sso/callback" $csghub.external }} + clientId: {{ generateClientId "Admin" }} + clientSecret: {{ generateClientSecret "Admin" }} + callbackUrl: {{ printf "%s/-/temporal/auth/sso/callback" $csghub.url }} scopes: - openid - email - profile + codec: endpoint: passAccessToken: false @@ -62,4 +62,4 @@ codec: forwardHeaders: -hideLogs: {{ $temporalUI.hideLogs }} \ No newline at end of file +hideLogs: {{ .temporal_ui.hideLogs }} diff --git a/opt/csghub/embedded/sv/accounting/templates/log/run.sample b/opt/csghub/embedded/sv/accounting/templates/log/run.sample index 32780476..c294c247 100755 --- a/opt/csghub/embedded/sv/accounting/templates/log/run.sample +++ b/opt/csghub/embedded/sv/accounting/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/accounting" }} -{{- if has (datasource "config").accounting "log" }} -{{- if has (datasource "config").accounting.log "dir" }} -{{- if (datasource "config").accounting.log.dir }} -{{- $logDir = (datasource "config").accounting.log.dir }} -{{- end }} +{{- if has .accounting "log" }} +{{- if .accounting.log.dir }} +{{- $logDir = .accounting.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/accounting/templates/run.sample b/opt/csghub/embedded/sv/accounting/templates/run.sample index 3717c755..31fa2409 100755 --- a/opt/csghub/embedded/sv/accounting/templates/run.sample +++ b/opt/csghub/embedded/sv/accounting/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("accounting" "launch" "-l" {{ $server.log.level | quote }}) +args=("accounting" "launch" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/aigateway/templates/log/run.sample b/opt/csghub/embedded/sv/aigateway/templates/log/run.sample index a1d1036a..c7246d9d 100755 --- a/opt/csghub/embedded/sv/aigateway/templates/log/run.sample +++ b/opt/csghub/embedded/sv/aigateway/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/aigateway" }} -{{- if has (datasource "config").aigateway "log" }} -{{- if has (datasource "config").aigateway.log "dir" }} -{{- if (datasource "config").aigateway.log.dir }} -{{- $logDir = (datasource "config").aigateway.log.dir }} -{{- end }} +{{- if has .aigateway "log" }} +{{- if .aigateway.log.dir }} +{{- $logDir = .aigateway.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/aigateway/templates/run.sample b/opt/csghub/embedded/sv/aigateway/templates/run.sample index d21bf3ae..47491ced 100755 --- a/opt/csghub/embedded/sv/aigateway/templates/run.sample +++ b/opt/csghub/embedded/sv/aigateway/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("aigateway" "launch" "-l" {{ $server.log.level | quote }}) +args=("aigateway" "launch" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/casdoor/templates/log/run.sample b/opt/csghub/embedded/sv/casdoor/templates/log/run.sample index e06ab5d2..3930983d 100755 --- a/opt/csghub/embedded/sv/casdoor/templates/log/run.sample +++ b/opt/csghub/embedded/sv/casdoor/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/casdoor" }} -{{- if has (datasource "config").casdoor "log" }} -{{- if has (datasource "config").casdoor.log "dir" }} -{{- if (datasource "config").casdoor.log.dir }} -{{- $logDir = (datasource "config").casdoor.log.dir }} -{{- end }} +{{- if has .casdoor "log" }} +{{- if .casdoor.log.dir }} +{{- $logDir = .casdoor.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/casdoor/templates/pre-start.sample b/opt/csghub/embedded/sv/casdoor/templates/pre-start.sample deleted file mode 100755 index 03247cc8..00000000 --- a/opt/csghub/embedded/sv/casdoor/templates/pre-start.sample +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# exit on error -set -e - -{{- $envDir := "/opt/csghub/etc/casdoor/env" }} -{{- $configDir := "/opt/csghub/etc/casdoor" }} - -{{- file.Write (printf "%s/DSN" $envDir) (tmpl.Exec "config.casdoor.db.dsn" .) }} - -{{- $db := tmpl.Exec "config.casdoor.db" . | data.YAML }} -{{- $postgresql := (datasource "config").postgresql }} -{{- $patroni := (datasource "config").patroni }} -{{- if or $postgresql.enable $patroni.enable }} -# Rename database casdoor to csghub_casdoor (If database casdoor exists) -/opt/csghub/bin/csghub-dbm -a rename -d "casdoor" -n {{ $db.dbname }} -# Create casdoor database (current default) -/opt/csghub/bin/csghub-dbm -a create -d {{ $db.dbname }} -u {{ $db.user }} -p {{ $db.password }} -# If casdoor database exists reset all objects owner to current db user -/opt/csghub/bin/csghub-dbm -a update -d {{ $db.dbname }} -N {{ $db.user }} -{{- end }} - -{{- $dataDir := (datasource "config").casdoor.data | default "/var/opt/csghub/casdoor" }} -if [ -f {{ $dataDir }}/app.conf ]; then - mkdir -p {{ $dataDir }}/conf || true - mv {{ $dataDir }}/app.conf {{ $dataDir }}/conf - cp -a {{ $configDir }}/{web,files} {{ $dataDir }}/ || true -fi - -COMMAND="/opt/csghub/embedded/bin/casdoor" - -LOGFILE={{ $dataDir }}/init.log -(cd {{ $dataDir }} && $COMMAND > "$LOGFILE" 2>&1 ) & -bg_pid=$! -echo "Started casdoor (PID $bg_pid) for initialization..." - -echo "Waiting for casdoor to reach Radius initialization stage..." -# Wait for specific log keyword: "Starting Radius server" -until grep -q "Starting Radius server" "$LOGFILE"; do - sleep 1 -done - -echo "Detected 'Starting Radius server' in casdoor logs; initialization is complete." - -# Force-stop the casdoor process now that initialization is done -echo "Checking for existing casdoor processes..." -if pgrep -f "$COMMAND" >/dev/null; then - echo "Found existing casdoor process, attempting graceful shutdown..." - pkill -f "$COMMAND" || true - - # Force kill if still running after graceful attempt - if pgrep -f "$COMMAND" >/dev/null; then - echo "Forcing termination of remaining processes..." - pkill -9 -f "$COMMAND" || true - fi -else - echo "No existing casdoor process found." -fi - -# Load local sql scripts -cd /opt/csghub/service/casdoor -[ -x "/opt/csghub/etc/csghub/scripts/pg_executor" ] && chpst -e {{ $envDir }} /opt/csghub/etc/csghub/scripts/pg_executor diff --git a/opt/csghub/embedded/sv/casdoor/templates/run.sample b/opt/csghub/embedded/sv/casdoor/templates/run.sample index ab6732dd..0f783d90 100755 --- a/opt/csghub/embedded/sv/casdoor/templates/run.sample +++ b/opt/csghub/embedded/sv/casdoor/templates/run.sample @@ -6,14 +6,53 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start +readonly ENV_DIR=/opt/csghub/service/casdoor/env + +{{ if or .postgresql.enable .patroni.enable }} +# Wait for PostgreSQL +echo "Waiting for PostgreSQL to become ready..." +while ! /opt/csghub/embedded/sv/postgresql/bin/pg_isready -q; do sleep 2; done +echo "PostgreSQL is ready." +DB_USER=$(cat "$ENV_DIR/DB_USER") +DB_PASSWORD=$(cat "$ENV_DIR/DB_PASSWORD") +DB_NAME=$(cat "$ENV_DIR/DB_NAME") +# Create database and update permissions for casdoor user +/opt/csghub/bin/csghub-dbm -a rename -d casdoor -n "$DB_NAME" +/opt/csghub/bin/csghub-dbm -a create -d "$DB_NAME" -u "$DB_USER" -p "$DB_PASSWORD" +/opt/csghub/bin/csghub-dbm -a update -d "$DB_NAME" -N "$DB_USER" +{{- end }} + +readonly CASDOOR_BIN=/opt/csghub/embedded/bin/casdoor +{{- $dataDir := .casdoor.data | default "/var/opt/csghub/casdoor" }} +cd {{ $dataDir }} +# Run casdoor in the background to perform initialization tasks, then shut it down once done. +LOGFILE={{ $dataDir }}/init.log +$CASDOOR_BIN > "$LOGFILE" 2>&1 & +bg_pid=$! +echo "Started casdoor (PID $bg_pid) for initialization..." +echo "Waiting for casdoor to reach Radius initialization stage..." +until grep -q "Starting Radius server" "$LOGFILE"; do + sleep 1 +done +echo "Detected 'Starting Radius server' in casdoor logs; initialization is complete." +# Force-stop the casdoor process now that initialization is done +echo "Checking for existing casdoor processes..." +if pgrep -f "$CASDOOR_BIN" >/dev/null; then + echo "Found existing casdoor process, attempting graceful shutdown..." + pkill -f "$CASDOOR_BIN" || true + # Force kill if still running after graceful attempt + if pgrep -f "$CASDOOR_BIN" >/dev/null; then + echo "Forcing termination of remaining processes..." + pkill -9 -f "$CASDOOR_BIN" || true + fi +else + echo "No existing casdoor process found." fi -{{- $dataDir := (datasource "config").casdoor.data | default "/var/opt/csghub/casdoor" }} +# Execute SQL seed scripts (after casdoor init so tables exist) +cd /opt/csghub/service/casdoor && chpst -e "$ENV_DIR" /opt/csghub/embedded/etc/csghub/scripts/pg_executor cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/casdoor/env -P \ - -u casdoor:casdoor \ +exec chpst -e /opt/csghub/service/casdoor/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ /opt/csghub/embedded/bin/casdoor diff --git a/opt/csghub/embedded/sv/casdoor/templates/scripts/update_casdoor.sql.sample b/opt/csghub/embedded/sv/casdoor/templates/scripts/update_casdoor.sql.sample index 5a53cf8f..af5a4b22 100644 --- a/opt/csghub/embedded/sv/casdoor/templates/scripts/update_casdoor.sql.sample +++ b/opt/csghub/embedded/sv/casdoor/templates/scripts/update_casdoor.sql.sample @@ -1,6 +1,6 @@ -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML }} -{{- $casdoorEndpoint := tmpl.Exec "endpoint.casdoor" . -}} -{{- $casdoorDB := tmpl.Exec "config.casdoor.db" . | data.YAML -}} +{{- $csghub := resolveUrl .csghub.external_url }} +{{- $casdoorEp := resolveEndpoint $csghub.scheme $csghub.host .casdoor.listen "127.0.0.1:8000" }} +{{- $casdoorDB := buildDBConfig .casdoor.postgresql }} -- -- Record Timestamp -- @@ -44,8 +44,8 @@ BEGIN UPDATE application SET - redirect_uris = ''["{{ $csghub.external }}/api/v1/callback/casdoor"]'', - logo = ''{{ $casdoorEndpoint }}''||substring(logo from ''(/files/avatar/.*)'') + redirect_uris = ''["{{ $csghub.url }}/api/v1/callback/casdoor"]'', + logo = ''{{ $casdoorEp.url }}''||substring(logo from ''(/files/avatar/.*)'') WHERE name in (''CSGHub'',''Csgship'',''CSGShip'') '); @@ -55,8 +55,8 @@ BEGIN UPDATE application SET - client_id = ''{{ tmpl.Exec "GenClientId" "CSGHub" }}'', - client_secret = ''{{ tmpl.Exec "GenClientSecret" "CSGHub" }}'' + client_id = ''{{ generateClientId "CSGHub" }}'', + client_secret = ''{{ generateClientSecret "CSGHub" }}'' WHERE name = ''CSGHub'' '); @@ -66,8 +66,8 @@ BEGIN UPDATE application SET - redirect_uris = ''["{{ $csghub.external }}/-/temporal/auth/sso/callback"]'', - logo = ''{{ $casdoorEndpoint }}''||substring(logo from ''(/files/avatar/.*)'') + redirect_uris = ''["{{ $csghub.url }}/-/temporal/auth/sso/callback"]'', + logo = ''{{ $casdoorEp.url }}''||substring(logo from ''(/files/avatar/.*)'') WHERE name = ''Admin'' '); @@ -77,8 +77,8 @@ BEGIN UPDATE application SET - client_id = ''{{ tmpl.Exec "GenClientId" "Admin" }}'', - client_secret = ''{{ tmpl.Exec "GenClientSecret" "Admin" }}'' + client_id = ''{{ generateClientId "Admin" }}'', + client_secret = ''{{ generateClientSecret "Admin" }}'' WHERE name = ''Admin'' '); @@ -88,8 +88,8 @@ BEGIN UPDATE application SET - client_id = ''{{ tmpl.Exec "GenClientId" "CSGShip" }}'', - client_secret = ''{{ tmpl.Exec "GenClientSecret" "CSGShip" }}'' + client_id = ''{{ generateClientId "CSGShip" }}'', + client_secret = ''{{ generateClientSecret "CSGShip" }}'' WHERE name = ''CSGShip'' '); @@ -99,7 +99,7 @@ BEGIN UPDATE application SET - form_side_html = regexp_replace(form_side_html, ''http://\d+\.\d+\.\d+\.\d+:\d+'', ''{{ $casdoorEndpoint }}'', ''g'') + form_side_html = regexp_replace(form_side_html, ''http://\d+\.\d+\.\d+\.\d+:\d+'', ''{{ $casdoorEp.url }}'', ''g'') WHERE name in (''CSGHub'',''Csgship'',''CSGShip'') '); @@ -131,10 +131,10 @@ BEGIN UPDATE organization SET - website_url = ''{{ $casdoorEndpoint }}'', - logo = ''{{ $casdoorEndpoint }}''||substring(logo from ''(/files/avatar/.*)''), - favicon = ''{{ $casdoorEndpoint }}''||substring(favicon from ''(/files/avatar/.*)''), - default_avatar = ''{{ $casdoorEndpoint }}''||substring(default_avatar from ''(/files/avatar/.*)'') + website_url = ''{{ $casdoorEp.url }}'', + logo = ''{{ $casdoorEp.url }}''||substring(logo from ''(/files/avatar/.*)''), + favicon = ''{{ $casdoorEp.url }}''||substring(favicon from ''(/files/avatar/.*)''), + default_avatar = ''{{ $casdoorEp.url }}''||substring(default_avatar from ''(/files/avatar/.*)'') WHERE name = ''OpenCSG'' '); @@ -143,20 +143,20 @@ END $$; -- Dynamically update user table (only if table exists) -{{- $initPassFile := "/etc/csghub/init_root_password" -}} -{{- $pass := "" -}} -{{- if file.Exists $initPassFile -}} - {{- $content := file.Read $initPassFile | trimSpace }} - {{- $parts := strings.Split ":" $content }} +{{- $initPassFile := "/etc/csghub/init_root_password" }} +{{- $pass := "" }} +{{- if fileExists $initPassFile }} + {{- $content := fileRead $initPassFile | trim }} + {{- $parts := splitList ":" $content }} {{- if gt (len $parts) 1 }} - {{- $pass = index $parts 1 -}} + {{- $pass = index $parts 1 }} {{- else }} - {{- $pass = $content -}} + {{- $pass = $content }} {{- end }} -{{- end -}} -{{- if not $pass -}} - {{- $pass = random.Alpha 24 -}} - {{- file.Write $initPassFile (printf "root:%s\n" $pass) -}} +{{- end }} +{{- if not $pass }} + {{- $pass = randomAlpha 24 }} + {{- fileWrite $initPassFile (printf "root:%s\n" $pass) }} {{- end }} DO $$ BEGIN @@ -169,7 +169,7 @@ BEGIN "user" SET password_type = ''bcrypt'', - password = ''{{ $pass | crypto.Bcrypt }}'' + password = ''{{ $pass | bcrypt }}'' WHERE name = ''admin'' AND password_type = ''plain'' @@ -188,7 +188,7 @@ BEGIN UPDATE "user" SET - avatar = ''{{ $casdoorEndpoint }}''||substring(avatar from ''(/files/avatar/.*)'') + avatar = ''{{ $casdoorEp.url }}''||substring(avatar from ''(/files/avatar/.*)'') WHERE avatar ~ ''.*/files/avatar/OpenCSG/.*\.png'' '; @@ -197,8 +197,7 @@ END $$; -- Dynamically update provider table (only if table exists) -{{- $csgship := (datasource "config").csgship }} -{{- $oauth := $csgship.oauth }} +{{- $oauth := .csgship.oauth }} DO $$ BEGIN IF EXISTS ( @@ -224,10 +223,10 @@ BEGIN UPDATE "provider" SET - custom_logo = ''{{ $casdoorEndpoint }}''||substring(custom_logo from ''(/files/avatar/.*)'') + custom_logo = ''{{ $casdoorEp.url }}''||substring(custom_logo from ''(/files/avatar/.*)'') WHERE custom_logo ~ ''.*/files/avatar/OpenCSG/.*\.png'' '; END IF; END -$$; \ No newline at end of file +$$; diff --git a/opt/csghub/embedded/sv/consul/templates/log/run.sample b/opt/csghub/embedded/sv/consul/templates/log/run.sample index 7803713a..f2d590e4 100755 --- a/opt/csghub/embedded/sv/consul/templates/log/run.sample +++ b/opt/csghub/embedded/sv/consul/templates/log/run.sample @@ -1,9 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/consul" }} -{{- if has (datasource "config").consul "log_dir" }} -{{- if (datasource "config").consul.log_dir }} -{{- $logDir = (datasource "config").consul.log_dir }} +{{- if has .consul "log_dir" }} +{{- if .consul.log_dir }} +{{- $logDir = .consul.log_dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/consul/templates/run.sample b/opt/csghub/embedded/sv/consul/templates/run.sample index 2c0e10cf..57471e1c 100755 --- a/opt/csghub/embedded/sv/consul/templates/run.sample +++ b/opt/csghub/embedded/sv/consul/templates/run.sample @@ -6,12 +6,12 @@ exec 2>&1 # Exit if execute with any errors set -e -{{- $dataDir := (datasource "config").consul.data_dir | default "/var/opt/csghub/consul" }} +{{- $dataDir := .consul.data_dir | default "/var/opt/csghub/consul" }} if [ -e {{ $dataDir }}/server_metadata.json ]; then rm {{ $dataDir }}/server_metadata.json || true fi cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/consul/env -P \ - -u consul:consul \ - /opt/csghub/embedded/bin/consul agent -config-dir={{ $dataDir }} +exec chpst -e /opt/csghub/service/consul/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/consul agent -config-dir=/opt/csghub/service/consul/etc diff --git a/opt/csghub/embedded/sv/dataviewer/templates/log/run.sample b/opt/csghub/embedded/sv/dataviewer/templates/log/run.sample index 3161e1d2..2fb1de3d 100755 --- a/opt/csghub/embedded/sv/dataviewer/templates/log/run.sample +++ b/opt/csghub/embedded/sv/dataviewer/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/dataviewer" }} -{{- if has (datasource "config").dataviewer "log" }} -{{- if has (datasource "config").dataviewer.log "dir" }} -{{- if (datasource "config").dataviewer.log.dir }} -{{- $logDir = (datasource "config").dataviewer.log.dir }} -{{- end }} +{{- if has .dataviewer "log" }} +{{- if .dataviewer.log.dir }} +{{- $logDir = .dataviewer.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/dataviewer/templates/run.sample b/opt/csghub/embedded/sv/dataviewer/templates/run.sample index 811cfcd2..6472504a 100755 --- a/opt/csghub/embedded/sv/dataviewer/templates/run.sample +++ b/opt/csghub/embedded/sv/dataviewer/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("dataviewer" "launch" "-l" {{ $server.log.level | quote }}) +args=("dataviewer" "launch" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/dnsmasq/templates/log/run.sample b/opt/csghub/embedded/sv/dnsmasq/templates/log/run.sample index e7873116..40a76bc1 100755 --- a/opt/csghub/embedded/sv/dnsmasq/templates/log/run.sample +++ b/opt/csghub/embedded/sv/dnsmasq/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/dnsmasq" }} -{{- if has (datasource "config").dnsmasq "log" }} -{{- if has (datasource "config").dnsmasq.log "dir" }} -{{- if (datasource "config").dnsmasq.log.dir }} -{{- $logDir = (datasource "config").dnsmasq.log.dir }} -{{- end }} +{{- if has .dnsmasq "log" }} +{{- if .dnsmasq.log.dir }} +{{- $logDir = .dnsmasq.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/dnsmasq/templates/pre-start.sample b/opt/csghub/embedded/sv/dnsmasq/templates/pre-start.sample deleted file mode 100755 index a7a246ad..00000000 --- a/opt/csghub/embedded/sv/dnsmasq/templates/pre-start.sample +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $configDir := "/opt/csghub/etc/dnsmasq" }} -{{- $runner := (datasource "config").runner }} -{{- $ns := $runner.namespace }} -{{- $services := $runner.knative.services }} -{{- range $services }} -{{- $entry := printf "address=/%s.%s/127.0.0.1" $ns .domain }} -{{- $configFile := printf "%s/%s.conf" $configDir .domain }} -{{- file.Write $configFile $entry }} -{{- end }} - -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -{{- if and $runner.use_public_domain (regexp.Match `^[a-zA-Z0-9.-]+$` $csghub.host) }} -{{- $rootDomain := tmpl.Exec "domain.root" . }} -{{- $entry := printf "cname=public.%s,%s" $rootDomain $csghub.host }} -{{- $configFile := printf "%s/%s.conf" $configDir (printf "public.%s" $rootDomain) }} -{{- file.Write $configFile $entry }} -{{- end }} - -# Enable local resolution -if [ -f "/etc/resolv.conf" ]; then - if ! grep -q -w '127.0.0.1' /etc/resolv.conf; then - cp -f /etc/resolv.conf /tmp/resolv.conf.bak - sed '0,/nameserver/s//nameserver 127.0.0.1\n&/' /tmp/resolv.conf.bak > /etc/resolv.conf - fi -fi - -# Grant dnsmasq permission to bind to port numbers below 1024 -# You can using `getcap /opt/csghub/embedded/sv/dnsmasq/bin/dnsmasq` verify -setcap 'cap_net_bind_service=+ep' /opt/csghub/embedded/sv/dnsmasq/bin/dnsmasq diff --git a/opt/csghub/embedded/sv/dnsmasq/templates/run.sample b/opt/csghub/embedded/sv/dnsmasq/templates/run.sample index 1650c34a..22e7d38c 100755 --- a/opt/csghub/embedded/sv/dnsmasq/templates/run.sample +++ b/opt/csghub/embedded/sv/dnsmasq/templates/run.sample @@ -6,11 +6,9 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi +# Grant dnsmasq permission to bind to privileged ports +setcap cap_net_bind_service=+ep /opt/csghub/embedded/sv/dnsmasq/bin/dnsmasq -exec chpst -e /opt/csghub/etc/dnsmasq/env -P \ - -u dnsmasq:dnsmasq \ - /opt/csghub/embedded/bin/dnsmasq --conf-dir=/opt/csghub/etc/dnsmasq --no-daemon +exec chpst -e /opt/csghub/service/dnsmasq/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/dnsmasq --conf-dir=/opt/csghub/embedded/etc/dnsmasq --no-daemon diff --git a/opt/csghub/embedded/sv/gitaly/templates/log/run.sample b/opt/csghub/embedded/sv/gitaly/templates/log/run.sample index 677164bc..a04ff078 100755 --- a/opt/csghub/embedded/sv/gitaly/templates/log/run.sample +++ b/opt/csghub/embedded/sv/gitaly/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/gitaly" }} -{{- if has (datasource "config").gitaly "logging" }} -{{- if has (datasource "config").gitaly.logging "dir" }} -{{- if (datasource "config").gitaly.logging.dir }} -{{- $logDir = (datasource "config").gitaly.logging.dir }} -{{- end }} +{{- if has .gitaly "logging" }} +{{- if .gitaly.logging.dir }} +{{- $logDir = .gitaly.logging.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/gitaly/templates/pre-start.sample b/opt/csghub/embedded/sv/gitaly/templates/pre-start.sample deleted file mode 100755 index d772f648..00000000 --- a/opt/csghub/embedded/sv/gitaly/templates/pre-start.sample +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -# Define the repository parent directory -{{- $gitaly := (datasource "config").gitaly }} -{{- $dataDir := $gitaly.data_dir | default "/var/opt/csghub/gitaly" }} -mkdir -m 0777 -p {{ $dataDir }}/run {{ $dataDir }}/repositories || true - -# Create pid file and variable file -{{- $pidFile := printf "%s/gitaly.pid" $dataDir }} -if [ ! -s {{ $pidFile }} ]; then - echo -n 0 > {{ $pidFile }} -fi - -{{- $envDir := "/opt/csghub/etc/gitaly/env" }} -{{- $pidFileEnv := printf "%s/GITALY_PID_FILE" $envDir }} -{{- file.Write $pidFileEnv $pidFile }} diff --git a/opt/csghub/embedded/sv/gitaly/templates/run.sample b/opt/csghub/embedded/sv/gitaly/templates/run.sample index ccf3c3a6..b58668fa 100755 --- a/opt/csghub/embedded/sv/gitaly/templates/run.sample +++ b/opt/csghub/embedded/sv/gitaly/templates/run.sample @@ -9,14 +9,10 @@ ulimit -n 15000 # Exit if execute with any errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -{{- $dataDir := (datasource "config").gitaly.data_dir | default "/var/opt/csghub/gitaly" }} +{{- $dataDir := .gitaly.data_dir | default "/var/opt/csghub/gitaly" }} +{{- $configDir := "/opt/csghub/service/gitaly/etc" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/gitaly/env -P \ - -u git:git \ - /opt/csghub/embedded/bin/gitaly-wrapper /opt/csghub/embedded/bin/gitaly serve {{ $dataDir }}/config.toml +exec chpst -e /opt/csghub/service/gitaly/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/gitaly-wrapper /opt/csghub/embedded/bin/gitaly serve {{ $configDir }}/config.toml diff --git a/opt/csghub/embedded/sv/gitlab_shell/templates/log/run.sample b/opt/csghub/embedded/sv/gitlab_shell/templates/log/run.sample index 473d99ea..81cbe46c 100755 --- a/opt/csghub/embedded/sv/gitlab_shell/templates/log/run.sample +++ b/opt/csghub/embedded/sv/gitlab_shell/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/gitlab_shell" }} -{{- if has (datasource "config").gitlab_shell "logging" }} -{{- if has (datasource "config").gitlab_shell.logging "dir" }} -{{- if (datasource "config").gitlab_shell.logging.dir }} -{{- $logDir = (datasource "config").gitlab_shell.logging.dir }} -{{- end }} +{{- if has .gitlab_shell "logging" }} +{{- if .gitlab_shell.logging.dir }} +{{- $logDir = .gitlab_shell.logging.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/gitlab_shell/templates/run.sample b/opt/csghub/embedded/sv/gitlab_shell/templates/run.sample index f91a5714..0b7a2307 100755 --- a/opt/csghub/embedded/sv/gitlab_shell/templates/run.sample +++ b/opt/csghub/embedded/sv/gitlab_shell/templates/run.sample @@ -9,11 +9,10 @@ set -e # Attempt to change ulimit before the set -e flag, ignore failures ulimit -n 15000 -# Run the pre-start script to ensure that the service meets the startup conditions -{{- $gitaly := (datasource "config").gitaly }} -{{- $dataDir := (datasource "config").gitlab_shell.data | default "/var/opt/csghub/gitlab_shell" }} -{{- $token := $gitaly.auth.token | default (crypto.PBKDF2 "gitaly" "opencsg" 2048 8) }} -{{- file.Write "/opt/csghub/etc/gitlab_shell/.gitlab_shell_secret" $token }} +{{- $dataDir := .gitlab_shell.data | default "/var/opt/csghub/gitlab_shell" }} +{{- $configDir := .gitlab_shell.config | default "/opt/csghub/service/gitlab_shell/etc" }} +{{- $token := .gitaly.auth.token | default (pbkdf2 "gitaly" "opencsg" 2048 8) }} +{{- fileWrite "/opt/csghub/service/gitlab_shell/.gitlab_shell_secret" $token }} # Exit if execute with any errors set -e @@ -39,11 +38,10 @@ else ssh-keygen -t ed25519 -f $KEYS_DIRECTORY/ssh_host_ed25519_key -N "" fi -{{- $dataDir := (datasource "config").gitlab_shell.data | default "/var/opt/csghub/gitlab_shell" }} if [ "${SSH_DAEMON}" == "gitlab-sshd" ]; then - exec chpst -e /opt/csghub/etc/gitlab_shell/env -P \ - -u git:git \ - /opt/csghub/embedded/bin/gitlab-sshd -config-dir {{ $dataDir }} + exec chpst -e /opt/csghub/service/gitlab_shell/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/gitlab-sshd -config-dir {{ $configDir }} else if [ "${USE_GITLAB_LOGGER-0}" -eq 1 ]; then /usr/bin/gitlab-logger /var/log/gitlab-shell & @@ -57,4 +55,4 @@ else fi exec /usr/sbin/sshd -D -E /var/log/gitlab-shell/gitlab-shell.log -fi \ No newline at end of file +fi diff --git a/opt/csghub/embedded/sv/logger/templates/pre-start.sample b/opt/csghub/embedded/sv/logger/templates/pre-start.sample deleted file mode 100755 index 63128ac8..00000000 --- a/opt/csghub/embedded/sv/logger/templates/pre-start.sample +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $envDir := "/opt/csghub/etc/minio/env" }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -{{- $logger := (datasource "config").logger }} -{{- file.Write (printf "%s/WEBHOOK_LISTEN_PORT" $envDir) $logger.listen_port }} -{{- file.Write (printf "%s/WEBHOOK_AUTH_TOKEN" $envDir) $logger.auth_token }} - diff --git a/opt/csghub/embedded/sv/logger/templates/run.sample b/opt/csghub/embedded/sv/logger/templates/run.sample index 1d82bfa2..1e4a85d0 100755 --- a/opt/csghub/embedded/sv/logger/templates/run.sample +++ b/opt/csghub/embedded/sv/logger/templates/run.sample @@ -6,11 +6,6 @@ exec 2>&1 # Exit if execute with any errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -exec chpst -e /opt/csghub/etc/minio/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/logger \ No newline at end of file +exec chpst -e /opt/csghub/service/minio/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/logger diff --git a/opt/csghub/embedded/sv/loki/templates/log/run.sample b/opt/csghub/embedded/sv/loki/templates/log/run.sample index fcafd54a..5e2ce358 100755 --- a/opt/csghub/embedded/sv/loki/templates/log/run.sample +++ b/opt/csghub/embedded/sv/loki/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/loki" }} -{{- if has (datasource "config").loki "log" }} -{{- if has (datasource "config").loki.log "dir" }} -{{- if (datasource "config").loki.log.dir }} -{{- $logDir = (datasource "config").loki.log.dir }} -{{- end }} +{{- if has .loki "log" }} +{{- if .loki.log.dir }} +{{- $logDir = .loki.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/loki/templates/run.sample b/opt/csghub/embedded/sv/loki/templates/run.sample index 9ef7e0e3..173bc3e3 100755 --- a/opt/csghub/embedded/sv/loki/templates/run.sample +++ b/opt/csghub/embedded/sv/loki/templates/run.sample @@ -3,18 +3,14 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # fail on errors set -e -{{- $dataDir := (datasource "config").loki.data | default "/var/opt/csghub/loki" }} +{{- $dataDir := .loki.data | default "/var/opt/csghub/loki" }} +{{- $configDir := "/opt/csghub/service/loki/etc" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/loki/env -P \ - -u loki:loki \ +exec chpst -e /opt/csghub/service/loki/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ /opt/csghub/embedded/bin/loki \ - -config.file={{ $dataDir }}/loki-config.yaml \ + -config.file={{ $configDir }}/loki-config.yaml \ -server.path-prefix="/-/loki" diff --git a/opt/csghub/embedded/sv/minio/templates/create_bucket.sample b/opt/csghub/embedded/sv/minio/templates/create_bucket.sample index 18fe4bed..d8a7723a 100644 --- a/opt/csghub/embedded/sv/minio/templates/create_bucket.sample +++ b/opt/csghub/embedded/sv/minio/templates/create_bucket.sample @@ -3,9 +3,8 @@ # MinIO bucket creation script # Usage: ./create_bucket -{{- $minio := (datasource "config").minio }} -{{- $user := $minio.auth.user }} -{{- $password := $minio.auth.password | default (crypto.PBKDF2 $user "opencsg" 2048 8) }} +{{- $user := .minio.auth.user }} +{{- $password := .minio.auth.password | default (pbkdf2 $user "opencsg" 2048 8) }} # Configuration MC_PATH="/opt/csghub/embedded/bin/mc" # Path to mc client diff --git a/opt/csghub/embedded/sv/minio/templates/log/run.sample b/opt/csghub/embedded/sv/minio/templates/log/run.sample index 8a06c008..f253e2f8 100755 --- a/opt/csghub/embedded/sv/minio/templates/log/run.sample +++ b/opt/csghub/embedded/sv/minio/templates/log/run.sample @@ -1,8 +1,8 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/minio" }} -{{- if (datasource "config").minio.log }} -{{- $logDir = (datasource "config").minio.log }} +{{- if .minio.log }} +{{- $logDir = .minio.log }} {{- end }} exec chpst -P \ diff --git a/opt/csghub/embedded/sv/minio/templates/ping_minio.sample b/opt/csghub/embedded/sv/minio/templates/ping_minio.sample new file mode 100644 index 00000000..e6745b47 --- /dev/null +++ b/opt/csghub/embedded/sv/minio/templates/ping_minio.sample @@ -0,0 +1,29 @@ +#!/bin/bash + +# MinIO readiness check script +# Usage: ./ping_minio +# Returns 0 if Minio is ready, 1 otherwise. + +{{- $user := .minio.auth.user }} +{{- $password := .minio.auth.password | default (pbkdf2 $user "opencsg" 2048 8) }} + +MC_PATH="/opt/csghub/embedded/bin/mc" +MINIO_ALIAS="myMinio" +MINIO_ENDPOINT="http://127.0.0.1:9000" +MINIO_ACCESS_KEY={{ $user }} +MINIO_SECRET_KEY={{ $password }} + +# Configure MinIO alias if not already set +if ! "$MC_PATH" alias list | grep -q "$MINIO_ALIAS"; then + "$MC_PATH" alias set "$MINIO_ALIAS" "$MINIO_ENDPOINT" "$MINIO_ACCESS_KEY" "$MINIO_SECRET_KEY" > /dev/null 2>&1 +fi + +# Check if Minio is ready (10s timeout per attempt) +timeout 10 "$MC_PATH" ping "$MINIO_ALIAS" -c 1 > /dev/null 2>&1 +if [ $? -eq 0 ]; then + echo "MinIO is ready" + exit 0 +else + echo "MinIO is not ready" + exit 1 +fi diff --git a/opt/csghub/embedded/sv/minio/templates/pre-start.sample b/opt/csghub/embedded/sv/minio/templates/pre-start.sample deleted file mode 100755 index 662e6fc7..00000000 --- a/opt/csghub/embedded/sv/minio/templates/pre-start.sample +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $envDir := "/opt/csghub/etc/minio/env" }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -# Render environments -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -{{- $minio := (datasource "config").minio }} -{{- with $minio.auth }} -{{- file.Write (printf "%s/MINIO_ROOT_USER" $envDir) .user }} -{{- file.Write (printf "%s/MINIO_ROOT_PASSWORD" $envDir) (.password | default (crypto.PBKDF2 .user "opencsg" 2048 8)) }} -{{- file.Write (printf "%s/MINIO_CONSOLE_ADDRESS" $envDir) (.password | default (crypto.PBKDF2 .user "opencsg" 2048 8)) }} -{{- file.Write (printf "%s/MINIO_BROWSER_REDIRECT_URL" $envDir) ($minio.external_url | default (printf "%s://%s:9000/console/" $csghub.scheme $csghub.host)) }} -{{- file.Write "/etc/csghub/init_minio_password" (printf "%s:%s\n" .user (.password | default (crypto.PBKDF2 .user "opencsg" 2048 8)) ) }} -{{- end }} - -{{- $logger := (datasource "config").logger }} -{{- if $logger.enable }} -{{- file.Write (printf "%s/MINIO_AUDIT_WEBHOOK_ENABLE_PRIMARY" $envDir) "on" }} -{{- file.Write (printf "%s/MINIO_AUDIT_WEBHOOK_ENDPOINT_PRIMARY" $envDir) "http://localhost:9002/minio-logger" }} -{{- file.Write (printf "%s/MINIO_AUDIT_WEBHOOK_AUTH_TOKEN_PRIMARY" $envDir) (printf "Bearer %s" $logger.auth_token) }} -{{- end }} - -{{- range $env, $value := $minio.environments }} -{{- $envFile := printf "%s/%s" $envDir $env }} -{{- file.Write $envFile $value }} -{{- end }} - diff --git a/opt/csghub/embedded/sv/minio/templates/run.sample b/opt/csghub/embedded/sv/minio/templates/run.sample index 2f8be799..a62e0194 100755 --- a/opt/csghub/embedded/sv/minio/templates/run.sample +++ b/opt/csghub/embedded/sv/minio/templates/run.sample @@ -6,19 +6,13 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -{{- $minio := (datasource "config").minio }} -{{- $dataDir := $minio.data | default "/var/opt/csghub/minio/data" }} +{{- $dataDir := .minio.data | default "/var/opt/csghub/minio/data" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/minio/env -P \ - -u minio:minio \ +exec chpst -e /opt/csghub/service/minio/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ /opt/csghub/embedded/bin/minio server \ - --address {{ $minio.listen }} \ - --console-address {{ $minio.console.address }} \ + --address {{ .minio.listen }} \ + --console-address {{ .minio.console.address }} \ --json \ - {{ $dataDir }} \ No newline at end of file + {{ $dataDir }} diff --git a/opt/csghub/embedded/sv/mirror_lfs/templates/log/run.sample b/opt/csghub/embedded/sv/mirror_lfs/templates/log/run.sample index 68298c62..bb67b74a 100755 --- a/opt/csghub/embedded/sv/mirror_lfs/templates/log/run.sample +++ b/opt/csghub/embedded/sv/mirror_lfs/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/mirror_lfs" }} -{{- if has (datasource "config").mirror_lfs "log" }} -{{- if has (datasource "config").mirror_lfs.log "dir" }} -{{- if (datasource "config").mirror_lfs.log.dir }} -{{- $logDir = (datasource "config").mirror_lfs.log.dir }} -{{- end }} +{{- if has .mirror_lfs "log" }} +{{- if .mirror_lfs.log.dir }} +{{- $logDir = .mirror_lfs.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/mirror_lfs/templates/run.sample b/opt/csghub/embedded/sv/mirror_lfs/templates/run.sample index 3c26cced..defe1ce8 100755 --- a/opt/csghub/embedded/sv/mirror_lfs/templates/run.sample +++ b/opt/csghub/embedded/sv/mirror_lfs/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("mirror" "lfs-sync" "-l" {{ $server.log.level | quote }}) +args=("mirror" "lfs-sync" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/mirror_repo/templates/log/run.sample b/opt/csghub/embedded/sv/mirror_repo/templates/log/run.sample index dc887c42..0d6764ca 100755 --- a/opt/csghub/embedded/sv/mirror_repo/templates/log/run.sample +++ b/opt/csghub/embedded/sv/mirror_repo/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/mirror_repo" }} -{{- if has (datasource "config").mirror_repo "log" }} -{{- if has (datasource "config").mirror_repo.log "dir" }} -{{- if (datasource "config").mirror_repo.log.dir }} -{{- $logDir = (datasource "config").mirror_repo.log.dir }} -{{- end }} +{{- if has .mirror_repo "log" }} +{{- if .mirror_repo.log.dir }} +{{- $logDir = .mirror_repo.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/mirror_repo/templates/run.sample b/opt/csghub/embedded/sv/mirror_repo/templates/run.sample index 6bc19323..66e8979b 100755 --- a/opt/csghub/embedded/sv/mirror_repo/templates/run.sample +++ b/opt/csghub/embedded/sv/mirror_repo/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("mirror" "repo-sync" "-l" {{ $server.log.level | quote }}) +args=("mirror" "repo-sync" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/moderation/templates/log/run.sample b/opt/csghub/embedded/sv/moderation/templates/log/run.sample index 6290e5a4..3c5f8fde 100755 --- a/opt/csghub/embedded/sv/moderation/templates/log/run.sample +++ b/opt/csghub/embedded/sv/moderation/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/moderation" }} -{{- if has (datasource "config").moderation "log" }} -{{- if has (datasource "config").moderation.log "dir" }} -{{- if (datasource "config").moderation.log.dir }} -{{- $logDir = (datasource "config").moderation.log.dir }} -{{- end }} +{{- if has .moderation "log" }} +{{- if .moderation.log.dir }} +{{- $logDir = .moderation.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/moderation/templates/run.sample b/opt/csghub/embedded/sv/moderation/templates/run.sample index b789825f..3eebdd6c 100755 --- a/opt/csghub/embedded/sv/moderation/templates/run.sample +++ b/opt/csghub/embedded/sv/moderation/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("moderation" "launch" "-l" {{ $server.log.level | quote }}) +args=("moderation" "launch" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/nats/templates/log/run.sample b/opt/csghub/embedded/sv/nats/templates/log/run.sample index 340c306e..f3aa0395 100755 --- a/opt/csghub/embedded/sv/nats/templates/log/run.sample +++ b/opt/csghub/embedded/sv/nats/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/nats" }} -{{- if has (datasource "config").nats "log" }} -{{- if has (datasource "config").nats.log "dir" }} -{{- if (datasource "config").nats.log.dir }} -{{- $logDir = (datasource "config").nats.log.dir }} -{{- end }} +{{- if has .nats "log" }} +{{- if .nats.log.dir }} +{{- $logDir = .nats.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/nats/templates/run.sample b/opt/csghub/embedded/sv/nats/templates/run.sample index 9406600c..090a8f15 100755 --- a/opt/csghub/embedded/sv/nats/templates/run.sample +++ b/opt/csghub/embedded/sv/nats/templates/run.sample @@ -6,9 +6,10 @@ exec 2>&1 # fail on errors set -e -{{- $dataDir := (datasource "config").nats.data | default "/var/opt/csghub/nats" }} +{{- $dataDir := .nats.data | default "/var/opt/csghub/nats" }} +{{- $configDir := "/opt/csghub/service/nats/etc" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/nats/env -P \ - -u nats:nats \ - /opt/csghub/embedded/bin/nats-server -c {{ $dataDir }}/nats.conf +exec chpst -e /opt/csghub/service/nats/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/nats-server -c {{ $configDir }}/nats.conf diff --git a/opt/csghub/embedded/sv/nginx/templates/control/h.sample b/opt/csghub/embedded/sv/nginx/templates/control/h.sample index f5f3612c..c9edb4ab 100644 --- a/opt/csghub/embedded/sv/nginx/templates/control/h.sample +++ b/opt/csghub/embedded/sv/nginx/templates/control/h.sample @@ -1,9 +1,9 @@ #!/bin/sh exec 1>&2 -{{- $dataDir := (datasource "config").nginx.data | default "/var/opt/csghub/nginx" }} +{{- $dataDir := .nginx.data | default "/var/opt/csghub/nginx" }} if pgrep nginx; then - chpst -e /opt/csghub/etc/nginx/env -P \ + chpst -e /opt/csghub/service/nginx/env -P \ -u root:root \ /opt/csghub/embedded/bin/nginx -c {{ $dataDir }}/nginx.conf -s reload fi diff --git a/opt/csghub/embedded/sv/nginx/templates/log/run.sample b/opt/csghub/embedded/sv/nginx/templates/log/run.sample index 33198a01..fed467a2 100755 --- a/opt/csghub/embedded/sv/nginx/templates/log/run.sample +++ b/opt/csghub/embedded/sv/nginx/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/nginx" }} -{{- if has (datasource "config").nginx "log" }} -{{- if has (datasource "config").nginx.log "dir" }} -{{- if (datasource "config").nginx.log.dir }} -{{- $logDir = (datasource "config").nginx.log.dir }} -{{- end }} +{{- if has .nginx "log" }} +{{- if .nginx.log.dir }} +{{- $logDir = .nginx.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/nginx/templates/pre-start.sample b/opt/csghub/embedded/sv/nginx/templates/pre-start.sample deleted file mode 100755 index 8471a110..00000000 --- a/opt/csghub/embedded/sv/nginx/templates/pre-start.sample +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $configDir := "/opt/csghub/etc/nginx" }} -{{- $dataDir := (datasource "config").nginx.data | default "/var/opt/csghub/nginx" }} -find {{ $configDir }} -type f ! -name '*.default' ! -name '*.sample' ! -name 'nginx.conf' ! -name 'env' ! -name '*.so' -exec cp -p {} {{ $dataDir }} \; - diff --git a/opt/csghub/embedded/sv/nginx/templates/run.sample b/opt/csghub/embedded/sv/nginx/templates/run.sample index 1a76944d..f46f663f 100755 --- a/opt/csghub/embedded/sv/nginx/templates/run.sample +++ b/opt/csghub/embedded/sv/nginx/templates/run.sample @@ -6,13 +6,8 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -{{- $dataDir := (datasource "config").nginx.data | default "/var/opt/csghub/nginx" }} +{{- $dataDir := .nginx.data | default "/var/opt/csghub/nginx" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/nginx/env -P \ - -u root:root \ - /opt/csghub/embedded/bin/nginx -c {{ $dataDir }}/nginx.conf -g "daemon off;" \ No newline at end of file +exec chpst -e /opt/csghub/service/nginx/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/nginx -c {{ $dataDir }}/nginx.conf -g "pid {{ $dataDir }}/nginx.pid; lock_file {{ $dataDir }}/nginx.lock; daemon off;" diff --git a/opt/csghub/embedded/sv/notifier/templates/log/run.sample b/opt/csghub/embedded/sv/notifier/templates/log/run.sample index 1ac2f4c0..8b45a3d9 100755 --- a/opt/csghub/embedded/sv/notifier/templates/log/run.sample +++ b/opt/csghub/embedded/sv/notifier/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/notifier" }} -{{- if has (datasource "config").notifier "log" }} -{{- if has (datasource "config").notifier.log "dir" }} -{{- if (datasource "config").notifier.log.dir }} -{{- $logDir = (datasource "config").notifier.log.dir }} -{{- end }} +{{- if has .notifier "log" }} +{{- if .notifier.log.dir }} +{{- $logDir = .notifier.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/notifier/templates/run.sample b/opt/csghub/embedded/sv/notifier/templates/run.sample index 54199a7d..143c9b91 100755 --- a/opt/csghub/embedded/sv/notifier/templates/run.sample +++ b/opt/csghub/embedded/sv/notifier/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("notification" "launch" "-l" {{ $server.log.level | quote }}) +args=("notification" "launch" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/patroni/templates/log/run.sample b/opt/csghub/embedded/sv/patroni/templates/log/run.sample index 0ebb9595..ff550b3a 100755 --- a/opt/csghub/embedded/sv/patroni/templates/log/run.sample +++ b/opt/csghub/embedded/sv/patroni/templates/log/run.sample @@ -1,9 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/patroni" }} -{{- if has (datasource "config").patroni "log_dir" }} -{{- if (datasource "config").patroni.log_dir }} -{{- $logDir = (datasource "config").patroni.log_dir }} +{{- if has .patroni "log_dir" }} +{{- if .patroni.log_dir }} +{{- $logDir = .patroni.log_dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/patroni/templates/pre-start.sample b/opt/csghub/embedded/sv/patroni/templates/pre-start.sample deleted file mode 100755 index ead72a9b..00000000 --- a/opt/csghub/embedded/sv/patroni/templates/pre-start.sample +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $envDir := "/opt/csghub/etc/patroni/env" }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -if [ -e /dev/watchdog ]; then - chown postgres:postgres /dev/watchdog || true -fi - -{{- $dataDir := (datasource "config").patroni.data_dir | default "/var/opt/csghub/postgresql" }} -[[ ! -d "{{ $dataDir }}" ]] && mkdir -p "{{ $dataDir }}" || true - -{{- file.Write (printf "%s/PGHOME" $envDir) "/opt/csghub/embedded/sv/postgresql" }} -{{- file.Write (printf "%s/PGDATA" $envDir) (printf "%s/data" $dataDir) }} -{{- file.Write (printf "%s/MANPATH" $envDir) "/opt/csghub/embedded/sv/postgresql/share/man:$MANPATH" }} -{{- file.Write (printf "%s/PYTHONHOME" $envDir) "/opt/csghub/embedded/python" }} -{{- file.Write (printf "%s/LD_LIBRARY_PATH" $envDir) "/opt/csghub/embedded/python/lib:/opt/csghub/embedded/sv/postgresql/lib:$LD_LIBRARY_PATH" }} -{{- file.Write (printf "%s/PATRONI_HOME" $envDir) "/opt/csghub/embedded/sv/patroni" }} -{{- file.Write (printf "%s/PYTHONPATH" $envDir) "/opt/csghub/embedded/sv/patroni/lib/python3.11/site-packages:/opt/csghub/embedded/python/lib/python3.11/site-packages" }} -{{- file.Write (printf "%s/PATH" $envDir) "/opt/csghub/embedded/python/bin:/opt/csghub/embedded/sv/postgresql/bin:$PATH" }} diff --git a/opt/csghub/embedded/sv/patroni/templates/run.sample b/opt/csghub/embedded/sv/patroni/templates/run.sample index e0b3bee5..f1b96120 100755 --- a/opt/csghub/embedded/sv/patroni/templates/run.sample +++ b/opt/csghub/embedded/sv/patroni/templates/run.sample @@ -6,15 +6,10 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -{{- $dataDir := (datasource "config").patroni.data_dir | default "/var/opt/csghub/patroni" }} -{{- $configDir := (datasource "config").patroni.config_dir | default "/var/opt/csghub/patroni" }} +{{- $dataDir := .patroni.data_dir | default "/var/opt/csghub/patroni" }} +{{- $configDir := .patroni.config_dir | default "/opt/csghub/service/patroni/etc" }} cd {{ $configDir }} -exec chpst -e /opt/csghub/etc/patroni/env -P \ - -u postgres:postgres \ - /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/bin/patroni {{ $configDir }}/patroni.yaml \ No newline at end of file +exec chpst -e /opt/csghub/service/patroni/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/python3.11 /opt/csghub/embedded/bin/patroni {{ $configDir }}/patroni.yaml diff --git a/opt/csghub/embedded/sv/portal/templates/log/run.sample b/opt/csghub/embedded/sv/portal/templates/log/run.sample index 591d52a5..bc1ca182 100755 --- a/opt/csghub/embedded/sv/portal/templates/log/run.sample +++ b/opt/csghub/embedded/sv/portal/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/portal" }} -{{- if has (datasource "config").portal "logging" }} -{{- if has (datasource "config").portal.logging "dir" }} -{{- if (datasource "config").portal.logging.dir }} -{{- $logDir = (datasource "config").portal.logging.dir }} -{{- end }} +{{- if has .portal "logging" }} +{{- if .portal.logging.dir }} +{{- $logDir = .portal.logging.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/portal/templates/pre-start.sample b/opt/csghub/embedded/sv/portal/templates/pre-start.sample deleted file mode 100755 index db0d6599..00000000 --- a/opt/csghub/embedded/sv/portal/templates/pre-start.sample +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $envDir := "/opt/csghub/etc/portal/env" }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -# Define the repository parent directory -{{- $portal := (datasource "config").portal }} -{{- $dataDir := $portal.data | default "/var/opt/csghub/portal" }} - -{{- file.Write (printf "%s/CSGHUB_PORTAL_APP_ENV" $envDir) "production" }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_ON_PREMISE" $envDir) "true" }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_SENSITIVE_CHECK" $envDir) "false" }} - -{{- $apiToken := tmpl.Exec "GenHubApiToken" . }} - -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_STARHUB_BASE_URL" $envDir) $csghub.external }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_STARHUB_API_KEY" $envDir) $apiToken }} - -{{- if eq $csghub.scheme "https" }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_ENABLE_HTTPS" $envDir) "true" }} -{{- else }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_ENABLE_HTTPS" $envDir) "false" }} -{{- end }} - -{{- $db := $portal.postgresql }} -{{- $password := $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) }} -{{- $dsn := printf "postgresql://%s:%s@%s:%d/%s?sslmode=disable" (conv.ToString $db.user) (conv.ToString $password) (conv.ToString $db.host) (conv.ToInt64 $db.port) (conv.ToString $db.name) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_DATABASE_DSN" $envDir) $dsn }} - -{{- $postgresql := (datasource "config").postgresql }} -{{- $patroni := (datasource "config").patroni }} -{{- if or $postgresql.enable $patroni.enable }} -# Create database and user -/opt/csghub/bin/csghub-dbm -d {{ $db.name }} -u {{ $db.user }} -p {{ $password }} -# If portal database exists reset all objects owner to current db user -/opt/csghub/bin/csghub-dbm -a update -d {{ $db.name }} -N {{ $db.user }} -{{- end }} - -{{- $casdoor := tmpl.Exec "config.casdoor.conn" . | data.YAML }} -{{- $casdoorClientId := tmpl.Exec "GenClientId" "CSGHub" }} -{{- $callback := printf "%s://%s:%s/login/oauth/authorize?client_id=%s&response_type=code&redirect_uri=%s/api/v1/callback/casdoor&scope=read&state=casdoor" $csghub.scheme $csghub.host $casdoor.port $casdoorClientId $csghub.external }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_LOGIN_URL" $envDir) $callback }} - -{{- $s3 := $portal.s3 }} -{{- $additionalS3 := $portal.additionalS3 }} -{{- $accessKeyId := $s3.access_key }} -{{- $accessSecretKey := $s3.secret_key | default (crypto.PBKDF2 $accessKeyId "opencsg" 2048 8) }} - -{{- file.Write (printf "%s/CSGHUB_PORTAL_S3_ENABLE_SSL" $envDir) $s3.secure }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_S3_REGION" $envDir) $s3.region }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_S3_ACCESS_KEY_ID" $envDir) $accessKeyId }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_S3_ACCESS_KEY_SECRET" $envDir) $accessSecretKey }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_S3_BUCKET" $envDir) $s3.bucket }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_S3_ENDPOINT" $envDir) ($s3.endpoint | default (printf "%s:9000" $csghub.host)) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_PRIVATE_S3_ENABLE_SSL" $envDir) ($additionalS3.secure | default $s3.secure) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_PRIVATE_S3_REGION" $envDir) ($additionalS3.region | default $s3.region) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_PRIVATE_S3_ACCESS_KEY_ID" $envDir) ($additionalS3.accessKeyId | default $accessKeyId) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_PRIVATE_S3_ACCESS_KEY_SECRET" $envDir) ($additionalS3.accessSecretKey | default $accessSecretKey) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_PRIVATE_S3_BUCKET" $envDir) ($additionalS3.bucket | default $s3.bucket) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_PRIVATE_S3_ENDPOINT" $envDir) ($additionalS3.endpoint | default $s3.endpoint) }} -{{- file.Write (printf "%s/CSGHUB_PORTAL_AI_GATEWAY_HOST" $envDir) (printf "%s/aigateway" $csghub.external) }} - -{{- $minio := (datasource "config").minio }} -{{- if $minio.enable }} -# Create bucket -[ -x "/opt/csghub/service/minio/create_bucket" ] && \ - /opt/csghub/service/minio/create_bucket {{ $portal.s3.bucket }} "public" && \ - /opt/csghub/service/minio/create_bucket {{ $portal.additionalS3.bucket }} -{{- end }} - diff --git a/opt/csghub/embedded/sv/portal/templates/run.sample b/opt/csghub/embedded/sv/portal/templates/run.sample index f1189aca..15b93cf4 100755 --- a/opt/csghub/embedded/sv/portal/templates/run.sample +++ b/opt/csghub/embedded/sv/portal/templates/run.sample @@ -9,25 +9,45 @@ ulimit -n 15000 # Exit if execute with any errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - +readonly ENV_DIR=/opt/csghub/service/portal/env + +{{ if or .postgresql.enable .patroni.enable }} +# Wait for PostgreSQL +echo "Waiting for PostgreSQL to become ready..." +while ! /opt/csghub/embedded/sv/postgresql/bin/pg_isready -q; do sleep 2; done +echo "PostgreSQL is ready." +DB_USER=$(cat "$ENV_DIR/DB_USER") +DB_PASSWORD=$(cat "$ENV_DIR/DB_PASSWORD") +DB_NAME=$(cat "$ENV_DIR/DB_NAME") +# Create database and update permissions for portal user +/opt/csghub/bin/csghub-dbm -a create -d "$DB_NAME" -u "$DB_USER" -p "$DB_PASSWORD" +/opt/csghub/bin/csghub-dbm -a update -d "$DB_NAME" -N "$DB_USER" +{{- end }} + +{{ if .minio.enable }} +# Wait for MinIO and create buckets +echo "Waiting for MinIO to become ready..." +while ! /opt/csghub/service/minio/ping_minio; do sleep 2; done +echo "MinIO is ready." +# Create both public and private buckets for portal +/opt/csghub/service/minio/create_bucket {{ .portal.s3.bucket | default "csghub-portal-public" }} public +/opt/csghub/service/minio/create_bucket {{ .portal.additionalS3.bucket | default "csghub-portal" }} +{{- end }} + +{{- $runUser := .run_user }} readonly PORTAL_BIN="/opt/csghub/embedded/bin/csghub-portal" -readonly ENV_DIR="/opt/csghub/etc/portal/env" echo "Initializing database migrations..." chpst -e "$ENV_DIR" -P \ - -u root:root \ + -u {{ $runUser }}:{{ $runUser }} \ "$PORTAL_BIN" migration init echo "Running database migrations..." chpst -e "$ENV_DIR" -P \ - -u root:root \ + -u {{ $runUser }}:{{ $runUser }} \ "$PORTAL_BIN" migration migrate echo "Starting portal..." exec chpst -e "$ENV_DIR" -P \ - -u root:root \ + -u {{ $runUser }}:{{ $runUser }} \ "$PORTAL_BIN" start server diff --git a/opt/csghub/embedded/sv/postgresql/templates/control/h.sample b/opt/csghub/embedded/sv/postgresql/templates/control/h.sample index 553c8c83..9f3fed9a 100644 --- a/opt/csghub/embedded/sv/postgresql/templates/control/h.sample +++ b/opt/csghub/embedded/sv/postgresql/templates/control/h.sample @@ -1,10 +1,9 @@ #!/bin/sh exec 1>&2 -{{- $dataDir := (datasource "config").postgresql.data_dir | default "/var/opt/csghub/postgresql/data" }} +{{- $dataDir := .postgresql.data_dir | default "/var/opt/csghub/postgresql/data" }} if [ -f "{{ $dataDir }}/postmaster.pid" ]; then - chpst -e /opt/csghub/etc/postgresql/env -P \ + chpst -e /opt/csghub/service/postgresql/env -P \ -u postgres:postgres \ /opt/csghub/embedded/bin/pg_ctl reload -D "{{ $dataDir }}" fi - diff --git a/opt/csghub/embedded/sv/postgresql/templates/control/t.sample b/opt/csghub/embedded/sv/postgresql/templates/control/t.sample index 753dc2cd..9a3491d9 100644 --- a/opt/csghub/embedded/sv/postgresql/templates/control/t.sample +++ b/opt/csghub/embedded/sv/postgresql/templates/control/t.sample @@ -1,9 +1,9 @@ #!/bin/sh exec 1>&2 -{{- $dataDir := (datasource "config").postgresql.data_dir | default "/var/opt/csghub/postgresql/data" }} +{{- $dataDir := .postgresql.data_dir | default "/var/opt/csghub/postgresql/data" }} if [ -f "{{ $dataDir }}/postmaster.pid" ]; then - chpst -e /opt/csghub/etc/postgresql/env -P \ + chpst -e /opt/csghub/service/postgresql/env -P \ -u postgres:postgres \ /opt/csghub/embedded/bin/pg_ctl stop -D "{{ $dataDir }}" -m fast fi @@ -33,4 +33,3 @@ if pgrep postgres >/dev/null; then else echo "No PostgreSQL processes running" fi - diff --git a/opt/csghub/embedded/sv/postgresql/templates/finish.sample b/opt/csghub/embedded/sv/postgresql/templates/finish.sample index 4fae5167..d6e8d165 100644 --- a/opt/csghub/embedded/sv/postgresql/templates/finish.sample +++ b/opt/csghub/embedded/sv/postgresql/templates/finish.sample @@ -1,5 +1,5 @@ #!/bin/sh exec 1>&2 -{{- $dataDir := (datasource "config").postgresql.data_dir | default "/var/opt/csghub/postgresql/data" }} +{{- $dataDir := .postgresql.data_dir | default "/var/opt/csghub/postgresql/data" }} rm -rf {{ $dataDir }}/postmaster.pid || true diff --git a/opt/csghub/embedded/sv/postgresql/templates/log/run.sample b/opt/csghub/embedded/sv/postgresql/templates/log/run.sample index ce049c45..0c50abb5 100755 --- a/opt/csghub/embedded/sv/postgresql/templates/log/run.sample +++ b/opt/csghub/embedded/sv/postgresql/templates/log/run.sample @@ -1,9 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/postgresql" }} -{{- if has (datasource "config").postgresql "log_dir" }} -{{- if (datasource "config").postgresql.log_dir }} -{{- $logDir = (datasource "config").postgresql.log_dir }} +{{- if has .postgresql "log_dir" }} +{{- if .postgresql.log_dir }} +{{- $logDir = .postgresql.log_dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/postgresql/templates/pre-start.sample b/opt/csghub/embedded/sv/postgresql/templates/pre-start.sample deleted file mode 100755 index 5032c009..00000000 --- a/opt/csghub/embedded/sv/postgresql/templates/pre-start.sample +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $envDir := "/opt/csghub/etc/postgresql/env" }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -{{- $postgresql := (datasource "config").postgresql }} -{{- $dataDir := $postgresql.data_dir | default "/var/opt/csghub/postgresql/data" }} - -{{- file.Write (printf "%s/PGHOME" $envDir) "/opt/csghub/embedded/sv/postgresql" }} -{{- file.Write (printf "%s/PGDATA" $envDir) $dataDir }} -{{- file.Write (printf "%s/LD_LIBRARY_PATH" $envDir) "$LD_LIBRARY_PATH:/opt/csghub/embedded/sv/postgresql/lib" }} -{{- file.Write (printf "%s/MANPATH" $envDir) "$MANPATH:/opt/csghub/embedded/sv/postgresql/share/man" }} -{{- file.Write (printf "%s/PATH" $envDir) "/opt/csghub/embedded/sv/postgresql/bin:$PATH" }} - -if [ ! -f "{{ $dataDir }}/PG_VERSION" ]; then - if [ -f {{ $dataDir }}/postgresql.conf ]; then - mv {{ $dataDir }}/*.conf {{ $dataDir }}/.. - fi - - echo "Initializing database cluster..." - chpst -e {{ $envDir }} -P \ - -u postgres:postgres \ - /opt/csghub/embedded/bin/initdb --encoding=UTF8 -D {{ $dataDir }} - - if [ -f {{ $dataDir }}/../postgresql.conf ]; then - mv {{ $dataDir }}/../*.conf {{ $dataDir }}/ - fi -fi - - diff --git a/opt/csghub/embedded/sv/postgresql/templates/run.sample b/opt/csghub/embedded/sv/postgresql/templates/run.sample index 255e6135..38b1210f 100755 --- a/opt/csghub/embedded/sv/postgresql/templates/run.sample +++ b/opt/csghub/embedded/sv/postgresql/templates/run.sample @@ -6,14 +6,21 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi +{{- $dataDir := .postgresql.data_dir | default "/var/opt/csghub/postgresql" }} -{{- $dataDir := (datasource "config").postgresql.data_dir | default "/var/opt/csghub/postgresql" }} +# Initialize database cluster if not already initialized +if [ ! -f "{{ $dataDir }}/PG_VERSION" ]; then + echo "Initializing database cluster..." + PARENT_DIR=$(dirname "{{ $dataDir }}") + [ -f "{{ $dataDir }}/postgresql.conf" ] && mv "{{ $dataDir }}"/*.conf "$PARENT_DIR/" + chpst -e /opt/csghub/service/postgresql/env -P -u postgres:postgres \ + /opt/csghub/embedded/bin/initdb --encoding=UTF8 -D "{{ $dataDir }}" + mv "$PARENT_DIR"/*.conf "{{ $dataDir }}/" 2>/dev/null || true +fi cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/postgresql/env -P \ - -u postgres:postgres \ - /opt/csghub/embedded/bin/postgres -D {{ $dataDir }} \ No newline at end of file +exec chpst -e /opt/csghub/service/postgresql/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/postgres -D {{ $dataDir }} \ + -c config_file=/opt/csghub/service/postgresql/etc/postgresql.conf \ + -c hba_file=/opt/csghub/service/postgresql/etc/pg_hba.conf diff --git a/opt/csghub/embedded/sv/praefect/templates/log/run.sample b/opt/csghub/embedded/sv/praefect/templates/log/run.sample index 31fa993c..3d1e8b91 100755 --- a/opt/csghub/embedded/sv/praefect/templates/log/run.sample +++ b/opt/csghub/embedded/sv/praefect/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/praefect" }} -{{- if has (datasource "config").praefect "logging" }} -{{- if has (datasource "config").praefect.logging "dir" }} -{{- if (datasource "config").praefect.logging.dir }} -{{- $logDir = (datasource "config").praefect.logging.dir }} -{{- end }} +{{- if has .praefect "logging" }} +{{- if .praefect.logging.dir }} +{{- $logDir = .praefect.logging.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/praefect/templates/pre-start.sample b/opt/csghub/embedded/sv/praefect/templates/pre-start.sample deleted file mode 100755 index da196eb4..00000000 --- a/opt/csghub/embedded/sv/praefect/templates/pre-start.sample +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $envDir := "/opt/csghub/etc/praefect/env" }} -{{- $dataDir := (datasource "config").praefect.data_dir | default "/var/opt/csghub/praefect" }} - -{{- $pidFile := printf "%s/praefect.pid" $dataDir }} -if [ ! -s {{ $pidFile }} ]; then - echo 0 > {{ $pidFile }} -fi -{{- file.Write (printf "%s/GITALY_PID_FILE" $envDir) $pidFile }} - diff --git a/opt/csghub/embedded/sv/praefect/templates/run.sample b/opt/csghub/embedded/sv/praefect/templates/run.sample index 572364b1..9114cda7 100755 --- a/opt/csghub/embedded/sv/praefect/templates/run.sample +++ b/opt/csghub/embedded/sv/praefect/templates/run.sample @@ -6,18 +6,13 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -{{- $dataDir := (datasource "config").praefect.data_dir | default "/var/opt/csghub/praefect" }} +{{- $dataDir := .praefect.data_dir | default "/var/opt/csghub/praefect" }} +{{- $configDir := "/opt/csghub/service/praefect/etc" }} cd {{ $dataDir }} -chpst -e /opt/csghub/etc/praefect/env -P \ - -u git:git \ - /opt/csghub/embedded/bin/praefect --config {{ $dataDir }}/config.toml sql-migrate - -exec chpst -e /opt/csghub/etc/praefect/env -P \ - -u git:git \ - /opt/csghub/embedded/bin/gitaly-wrapper /opt/csghub/embedded/bin/praefect -config {{ $dataDir }}/config.toml +chpst -e /opt/csghub/service/praefect/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/praefect --config {{ $configDir }}/config.toml sql-migrate +exec chpst -e /opt/csghub/service/praefect/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/gitaly-wrapper /opt/csghub/embedded/bin/praefect -config {{ $configDir }}/config.toml diff --git a/opt/csghub/embedded/sv/prometheus/templates/log/run.sample b/opt/csghub/embedded/sv/prometheus/templates/log/run.sample index cf0fa049..29cc3f39 100755 --- a/opt/csghub/embedded/sv/prometheus/templates/log/run.sample +++ b/opt/csghub/embedded/sv/prometheus/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/prometheus" }} -{{- if has (datasource "config").prometheus "log" }} -{{- if has (datasource "config").prometheus.log "dir" }} -{{- if (datasource "config").prometheus.log.dir }} -{{- $logDir = (datasource "config").prometheus.log.dir }} -{{- end }} +{{- if has .prometheus "log" }} +{{- if .prometheus.log.dir }} +{{- $logDir = .prometheus.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/prometheus/templates/run.sample b/opt/csghub/embedded/sv/prometheus/templates/run.sample index 84a16a21..93708a10 100755 --- a/opt/csghub/embedded/sv/prometheus/templates/run.sample +++ b/opt/csghub/embedded/sv/prometheus/templates/run.sample @@ -3,28 +3,23 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - # fail on errors set -e -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML }} -{{- $prometheus := (datasource "config").prometheus }} -{{- $dataDir := $prometheus.data | default "/var/opt/csghub/prometheus" }} -{{- $tsdb := $prometheus.tsdb }} +{{- $csghub := resolveUrl .csghub.external_url }} +{{- $dataDir := .prometheus.data | default "/var/opt/csghub/prometheus" }} +{{- $configDir := "/opt/csghub/service/prometheus/etc" }} +{{- $tsdb := .prometheus.tsdb }} {{- $tsdbPath := ($tsdb.path | default (printf "%s/data" $dataDir)) }} {{- $tsdbRetention := ($tsdb.retention | default "15d") }} -{{- $listenAddr := ($prometheus.listen | default "127.0.0.1:9090") }} -{{- $externalUrl := ($prometheus.external_url | default (printf "%s/-/prometheus" $csghub.external)) }} +{{- $listenAddr := (.prometheus.listen | default "127.0.0.1:9090") }} +{{- $externalUrl := (.prometheus.external_url | default (printf "%s/-/prometheus" $csghub.url)) }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/prometheus/env -P \ - -u prometheus:prometheus \ +exec chpst -e /opt/csghub/service/prometheus/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ /opt/csghub/embedded/bin/prometheus \ - --config.file={{ $dataDir }}/prometheus.yml \ + --config.file={{ $configDir }}/prometheus.yml \ --storage.tsdb.path={{ $tsdbPath }} \ --storage.tsdb.retention.time={{ $tsdbRetention }} \ --web.listen-address={{ $listenAddr }} \ @@ -32,4 +27,3 @@ exec chpst -e /opt/csghub/etc/prometheus/env -P \ --web.external-url={{ $externalUrl }} \ {{- end }} --log.level=info - diff --git a/opt/csghub/embedded/sv/redis/templates/log/run.sample b/opt/csghub/embedded/sv/redis/templates/log/run.sample index 9de99056..5e5428ad 100755 --- a/opt/csghub/embedded/sv/redis/templates/log/run.sample +++ b/opt/csghub/embedded/sv/redis/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/redis" }} -{{- if has (datasource "config").redis "log" }} -{{- if has (datasource "config").redis.log "dir" }} -{{- if (datasource "config").redis.log.dir }} -{{- $logDir = (datasource "config").redis.log.dir }} -{{- end }} +{{- if has .redis "log" }} +{{- if .redis.log.dir }} +{{- $logDir = .redis.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/redis/templates/pre-start.sample b/opt/csghub/embedded/sv/redis/templates/pre-start.sample deleted file mode 100755 index 62e82f24..00000000 --- a/opt/csghub/embedded/sv/redis/templates/pre-start.sample +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $dataDir := (datasource "config").redis.data | default "/var/opt/csghub/redis" }} - diff --git a/opt/csghub/embedded/sv/redis/templates/run.sample b/opt/csghub/embedded/sv/redis/templates/run.sample index 6db151e0..699d6cf4 100755 --- a/opt/csghub/embedded/sv/redis/templates/run.sample +++ b/opt/csghub/embedded/sv/redis/templates/run.sample @@ -6,14 +6,10 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -{{- $dataDir := (datasource "config").redis.data | default "/var/opt/csghub/redis" }} +{{- $dataDir := .redis.data | default "/var/opt/csghub/redis" }} +{{- $configDir := "/opt/csghub/service/redis/etc" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/redis/env -P \ - -u redis:redis \ - /opt/csghub/embedded/bin/redis-server {{ $dataDir }}/redis.conf +exec chpst -e /opt/csghub/service/redis/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/redis-server {{ $configDir }}/redis.conf diff --git a/opt/csghub/embedded/sv/registry/templates/log/run.sample b/opt/csghub/embedded/sv/registry/templates/log/run.sample index 3af4d82d..bf615ddc 100755 --- a/opt/csghub/embedded/sv/registry/templates/log/run.sample +++ b/opt/csghub/embedded/sv/registry/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/registry" }} -{{- if has (datasource "config").registry "log" }} -{{- if has (datasource "config").registry.log "dir" }} -{{- if (datasource "config").registry.log.dir }} -{{- $logDir = (datasource "config").registry.log.dir }} -{{- end }} +{{- if has .registry "log" }} +{{- if .registry.log.dir }} +{{- $logDir = .registry.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/registry/templates/pre-start.sample b/opt/csghub/embedded/sv/registry/templates/pre-start.sample deleted file mode 100755 index 809f13d8..00000000 --- a/opt/csghub/embedded/sv/registry/templates/pre-start.sample +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $dataDir := (datasource "config").registry.data | default "/var/opt/csghub/registry" }} - -{{- $registry := (datasource "config").registry }} -{{- $auth := $registry.auth }} -{{- $user := $auth.username | default "registry" }} -{{- $password := $auth.password | default (crypto.PBKDF2 "registry" "opencsg" 2048 8) }} -{{- if and $user $password}} -{{- $htpasswdFile := printf "%s/.htpasswd" $dataDir }} -{{- $encrypt := printf "%s:%s" $user (crypto.Bcrypt $password) }} -{{- file.Write $htpasswdFile $encrypt }} -{{- file.Write "/etc/csghub/init_registry_password" (printf "%s:%s\n" $user $password) }} -{{- end }} - -{{- $minio := (datasource "config").minio }} -{{- if $minio.enable }} -# Create bucket -[ -x "/opt/csghub/service/minio/create_bucket" ] && /opt/csghub/service/minio/create_bucket {{ $registry.storage.s3.bucket }} -{{- end }} - diff --git a/opt/csghub/embedded/sv/registry/templates/run.sample b/opt/csghub/embedded/sv/registry/templates/run.sample index b7172f24..a0949a09 100755 --- a/opt/csghub/embedded/sv/registry/templates/run.sample +++ b/opt/csghub/embedded/sv/registry/templates/run.sample @@ -6,15 +6,20 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi +{{ if .minio.enable }} +# Wait for MinIO and create bucket +echo "Waiting for MinIO to become ready..." +while ! /opt/csghub/service/minio/ping_minio; do sleep 2; done +echo "MinIO is ready." +S3_BUCKET=$(cat /opt/csghub/service/registry/env/S3_BUCKET) +# Create bucket for registry +/opt/csghub/service/minio/create_bucket "$S3_BUCKET" +{{- end }} -{{- $dataDir := (datasource "config").registry.data | default "/var/opt/csghub/registry" }} +{{- $dataDir := .registry.data | default "/var/opt/csghub/registry" }} +{{- $configDir := "/opt/csghub/service/registry/etc" }} cd {{ $dataDir }} -exec chpst -e /opt/csghub/etc/registry/env -P \ - -u registry:registry \ - /opt/csghub/embedded/bin/registry serve {{ $dataDir }}/config.yml - +exec chpst -e /opt/csghub/service/registry/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/registry serve {{ $configDir }}/config.yml diff --git a/opt/csghub/embedded/sv/rproxy/templates/log/run.sample b/opt/csghub/embedded/sv/rproxy/templates/log/run.sample index a163238a..86c07978 100755 --- a/opt/csghub/embedded/sv/rproxy/templates/log/run.sample +++ b/opt/csghub/embedded/sv/rproxy/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/rproxy" }} -{{- if has (datasource "config").rproxy "log" }} -{{- if has (datasource "config").rproxy.log "dir" }} -{{- if (datasource "config").rproxy.log.dir }} -{{- $logDir = (datasource "config").rproxy.log.dir }} -{{- end }} +{{- if has .rproxy "log" }} +{{- if .rproxy.log.dir }} +{{- $logDir = .rproxy.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/rproxy/templates/run.sample b/opt/csghub/embedded/sv/rproxy/templates/run.sample index d8d9b901..00510bc4 100755 --- a/opt/csghub/embedded/sv/rproxy/templates/run.sample +++ b/opt/csghub/embedded/sv/rproxy/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("start" "rproxy" "-l" {{ $server.log.level | quote }}) +args=("start" "rproxy" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/server/templates/log/run.sample b/opt/csghub/embedded/sv/server/templates/log/run.sample index f6b30ca3..895b6f40 100755 --- a/opt/csghub/embedded/sv/server/templates/log/run.sample +++ b/opt/csghub/embedded/sv/server/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/server" }} -{{- if has (datasource "config").server "log" }} -{{- if has (datasource "config").server.log "dir" }} -{{- if (datasource "config").server.log.dir }} -{{- $logDir = (datasource "config").server.log.dir }} -{{- end }} +{{- if has .server "log" }} +{{- if .server.log.dir }} +{{- $logDir = .server.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/server/templates/pre-start.sample b/opt/csghub/embedded/sv/server/templates/pre-start.sample deleted file mode 100755 index 335a328e..00000000 --- a/opt/csghub/embedded/sv/server/templates/pre-start.sample +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# exit when error -set -e - -{{- $configDir := "/opt/csghub/etc/server" }} -{{- $envDir := printf "%s/env" $configDir }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -{{- file.Write (printf "%s/GIN_MODE" $envDir) "release" }} - -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $db := $server.postgresql }} -{{- $password := $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) }} -{{- file.Write (printf "%s/POSTGRES_USER" $envDir) $db.user }} -{{- file.Write (printf "%s/POSTGRES_PASSWORD" $envDir) $password }} -{{- file.Write (printf "%s/POSTGRES_DB" $envDir) $db.name }} -{{- file.Write (printf "%s/HOME" $envDir) "/etc/csghub" }} -{{- file.Write (printf "%s/DSN" $envDir) (printf "postgresql://%s:%s@%s:%d/%s" (conv.ToString $db.user) (conv.ToString $password) (conv.ToString $db.host) (conv.ToInt64 $db.port) (conv.ToString $db.name)) }} -{{- file.Write (printf "%s/STARHUB_SERVER_PUBLIC_ROOT_DOMAIN" $envDir) " " }} -{{- file.Write (printf "%s/DUCKDB_HOME" $envDir) "/opt/csghub/embedded/sv/server" }} -{{- file.Write (printf "%s/DUCKDB_EXTENSION_DIRECTORY" $envDir) "/opt/csghub/embedded/sv/server" }} -{{- file.Write (printf "%s/STARHUB_SERVER_MIRROR_REMOTE" $envDir) "false" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_KUBE_SCHEDULER" $envDir) "volcano" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_NODE_RESOURCE_NAME" $envDir) "volcano.sh/node-vgpu-register" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_POD_RESOURCE_NAME" $envDir) "volcano.sh/vgpu-ids-new" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_RESOURCE_REQ_KEY" $envDir) "volcano.sh/vgpu-number" }} -{{- file.Write (printf "%s/STARHUB_SERVER_RUNNER_VGPU_MEMORY_REQ_KEY" $envDir) "volcano.sh/vgpu-memory" }} - -{{- $postgresql := (datasource "config").postgresql }} -{{- $patroni := (datasource "config").patroni }} -{{- if or $postgresql.enable $patroni.enable }} -# Create server database (current default) -/opt/csghub/bin/csghub-dbm -a create -d {{ $db.name }} -u {{ $db.user }} -p {{ $password }} -/opt/csghub/bin/csghub-dbm -a promote -u {{ $db.user }} -P SUPERUSER -# If server database exists reset all objects owner to current db user -/opt/csghub/bin/csghub-dbm -a update -d {{ $db.name }} -N {{ $db.user }} -{{- end }} - -{{- $dataDir := (datasource "config").server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} - -# Define constants -readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" - -# Build command arguments -## Migration init -args=("migration" "init") -[[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") - -# Database init -cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" || true - -## Migration migrate -args=("migration" "migrate") -[[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") - -# Database setup -cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" || true - -# Load local sql scripts -cd /opt/csghub/service/server -[ -x "/opt/csghub/etc/csghub/scripts/pg_executor" ] && chpst -e {{ $envDir }} /opt/csghub/etc/csghub/scripts/pg_executor - -{{- $minio := (datasource "config").minio }} -{{- if $minio.enable }} -# Create bucket -[ -x "/opt/csghub/service/minio/create_bucket" ] && \ - /opt/csghub/service/minio/create_bucket {{ $server.s3.bucket }} && \ - /opt/csghub/service/minio/create_bucket "csghub-runner" -{{- end }} - -find {{ $configDir }} -mindepth 1 -maxdepth 1 ! -name '*.sample' ! -name 'seed' ! -name 'env' ! -name 'scripts' -exec cp -r {} {{ $dataDir }} \; -echo "copy configuration files done." - -{{- $multi_sync := $server.multi_sync -}} -{{- if $multi_sync.enabled }} - -{{- $logDir := "/var/log/csghub/server" }} -{{- if has (datasource "config").server "log" }} -{{- if has (datasource "config").server.log "dir" }} -{{- if (datasource "config").server.log.dir }} -{{- $logDir = (datasource "config").server.log.dir }} -{{- end }} -{{- end }} -{{- end }} - -# Check and kill any existing multi-sync processes before starting a new one -if pgrep -f "$SERVER_BIN sync sync-as-client" >/dev/null; then - pkill -f "$SERVER_BIN sync sync-as-client" - # Wait a moment for the process to terminate - sleep 1 -fi - -# Start multi-sync -args=("sync" "sync-as-client") -[[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") - -# Initialize multi-source synchronization -(cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" > {{ $logDir }}/multi-sync.log 2>&1)& -{{- end }} diff --git a/opt/csghub/embedded/sv/server/templates/run.sample b/opt/csghub/embedded/sv/server/templates/run.sample index fecb35e3..eb1f2dd7 100755 --- a/opt/csghub/embedded/sv/server/templates/run.sample +++ b/opt/csghub/embedded/sv/server/templates/run.sample @@ -6,23 +6,61 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +readonly ENV_DIR=/opt/csghub/service/server/env +readonly SERVER_BIN=/opt/csghub/embedded/bin/csghub-server +CONFIG_ARG="" +[ -f "{{ $configFile }}" ] && CONFIG_ARG="--config={{ $configFile }}" -# Define constants -readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +{{ if or .postgresql.enable .patroni.enable }} +# Wait for PostgreSQL +echo "Waiting for PostgreSQL to become ready..." +while ! /opt/csghub/embedded/sv/postgresql/bin/pg_isready -q; do sleep 2; done +echo "PostgreSQL is ready." +DB_USER=$(cat "$ENV_DIR/POSTGRES_USER") +DB_PASSWORD=$(cat "$ENV_DIR/POSTGRES_PASSWORD") +DB_NAME=$(cat "$ENV_DIR/POSTGRES_DB") +# Create database and update permissions for server user +/opt/csghub/bin/csghub-dbm -a create -d "$DB_NAME" -u "$DB_USER" -p "$DB_PASSWORD" +/opt/csghub/bin/csghub-dbm -a promote -u "$DB_USER" -P SUPERUSER +/opt/csghub/bin/csghub-dbm -a update -d "$DB_NAME" -N "$DB_USER" +{{- end }} + +# Run server migrations before SQL seed scripts so tables exist (tolerate failure) +echo "Running server migration init..." +cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" $CONFIG_ARG migration init || true +echo "Running server migration migrate..." +cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" $CONFIG_ARG migration migrate || true + +# Execute SQL seed scripts (after migration so tables exist) +cd /opt/csghub/service/server && chpst -e "$ENV_DIR" /opt/csghub/embedded/etc/csghub/scripts/pg_executor + +{{ if .minio.enable }} +# Wait for MinIO and create buckets +echo "Waiting for MinIO to become ready..." +while ! /opt/csghub/service/minio/ping_minio; do sleep 2; done +echo "MinIO is ready." +# Create bucket for server +/opt/csghub/service/minio/create_bucket {{ .server.s3.bucket | default "csghub-server" }} +/opt/csghub/service/minio/create_bucket csghub-runner +{{- end }} + +{{ if .server.multi_sync.enabled }} +# Start multi-sync in background +pkill -f "csghub-server sync sync-as-client" 2>/dev/null || true +sleep 1 +cd {{ $dataDir }} && chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" $CONFIG_ARG sync sync-as-client \ + > /var/log/csghub/server/multi-sync.log 2>&1 & +echo "Started multi-sync (PID $!)" +{{- end }} # Build command arguments -args=("start" "server" "-l" {{ $server.log.level | quote }}) +args=("start" "server" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/temporal/templates/log/run.sample b/opt/csghub/embedded/sv/temporal/templates/log/run.sample index d1d815ad..78ec7c4d 100755 --- a/opt/csghub/embedded/sv/temporal/templates/log/run.sample +++ b/opt/csghub/embedded/sv/temporal/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/temporal" }} -{{- if has (datasource "config").temporal "log" }} -{{- if has (datasource "config").temporal.log "dir" }} -{{- if (datasource "config").temporal.log.dir }} -{{- $logDir = (datasource "config").temporal.log.dir }} -{{- end }} +{{- if has .temporal "log" }} +{{- if .temporal.log.dir }} +{{- $logDir = .temporal.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/temporal/templates/pre-start.sample b/opt/csghub/embedded/sv/temporal/templates/pre-start.sample deleted file mode 100755 index 4a37df54..00000000 --- a/opt/csghub/embedded/sv/temporal/templates/pre-start.sample +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Redirect stderr -> stdout -exec 2>&1 - -# Exit if execute with any errors -set -e - -{{- $dataDir := (datasource "config").temporal.data | default "/var/opt/csghub/temporal" }} -{{- $envDir := printf "%s/env" $dataDir }} - -# Clear all environment variable files -# rm -rf {{ $envDir }}/* || true - -{{- $temporal := (datasource "config").temporal }} -{{- $db := $temporal.postgresql }} -{{- $password := $db.password | default (crypto.PBKDF2 $db.user "opencsg" 2048 8) }} -{{- file.Write (printf "%s/DB" $envDir) "postgres12" }} -{{- file.Write (printf "%s/DBNAME" $envDir) $db.name }} -{{- file.Write (printf "%s/POSTGRES_SEEDS" $envDir) $db.host }} -{{- file.Write (printf "%s/DB_PORT" $envDir) $db.port }} -{{- file.Write (printf "%s/POSTGRES_USER" $envDir) $db.user }} -{{- file.Write (printf "%s/POSTGRES_PWD" $envDir) $password }} -{{- file.Write (printf "%s/DSN" $envDir) (printf "postgresql://%s:%s@%s:%d/%s" (conv.ToString $db.user) (conv.ToString $password) (conv.ToString $db.host) (conv.ToInt64 $db.port) (conv.ToString $db.name)) }} -{{- $visibilityDB := printf "%s_%s" $db.name $temporal.persistence.visibility_store }} -{{- file.Write (printf "%s/VISIBILITY_DBNAME" $envDir) $visibilityDB }} - -{{- $postgresql := (datasource "config").postgresql }} -{{- $patroni := (datasource "config").patroni }} -{{- if or $postgresql.enable $patroni.enable }} -# Rename database temporal to csghub_temporal (If database temporal exists) -/opt/csghub/bin/csghub-dbm -a rename -d "temporal" -n {{ $db.name }} -/opt/csghub/bin/csghub-dbm -a rename -d "temporal_visibility" -n {{ $visibilityDB }} -# Create temporal database (current default) -/opt/csghub/bin/csghub-dbm -a create -d {{ $db.name }} -u {{ $db.user }} -p {{ $password }} -/opt/csghub/bin/csghub-dbm -a promote -u {{ $db.user }} -P CREATEDB -# If temporal database exists reset all objects owner to current db user -/opt/csghub/bin/csghub-dbm -a update -d {{ $db.name }} -N {{ $db.user }} -/opt/csghub/bin/csghub-dbm -a update -d {{ $visibilityDB }} -N {{ $db.user }} -{{- end }} - -# Load local sql scripts -cd /opt/csghub/service/temporal -[ -x "/opt/csghub/etc/csghub/scripts/pg_executor" ] && chpst -e {{ $envDir }} /opt/csghub/etc/csghub/scripts/pg_executor diff --git a/opt/csghub/embedded/sv/temporal/templates/run.sample b/opt/csghub/embedded/sv/temporal/templates/run.sample index a1a7924a..d693ee46 100755 --- a/opt/csghub/embedded/sv/temporal/templates/run.sample +++ b/opt/csghub/embedded/sv/temporal/templates/run.sample @@ -3,27 +3,45 @@ # Redirect stderr -> stdout exec 2>&1 -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi +readonly ENV_DIR=/opt/csghub/service/temporal/env -: "${TEMPORAL_HOME:=/opt/csghub/etc/temporal}" +{{ if or .postgresql.enable .patroni.enable }} +# Wait for PostgreSQL +echo "Waiting for PostgreSQL to become ready..." +while ! /opt/csghub/embedded/sv/postgresql/bin/pg_isready -q; do sleep 2; done +echo "PostgreSQL is ready." +DB_USER=$(cat "$ENV_DIR/POSTGRES_USER") +DB_PASSWORD=$(cat "$ENV_DIR/POSTGRES_PWD") +DB_NAME=$(cat "$ENV_DIR/DBNAME") +VISIBILITY_DB_NAME=$(cat "$ENV_DIR/VISIBILITY_DBNAME") +# Database operations +/opt/csghub/bin/csghub-dbm -a rename -d temporal -n "$DB_NAME" +/opt/csghub/bin/csghub-dbm -a rename -d temporal_visibility -n "$VISIBILITY_DB_NAME" +/opt/csghub/bin/csghub-dbm -a create -d "$DB_NAME" -u "$DB_USER" -p "$DB_PASSWORD" +/opt/csghub/bin/csghub-dbm -a promote -u "$DB_USER" -P CREATEDB +/opt/csghub/bin/csghub-dbm -a update -d "$DB_NAME" -N "$DB_USER" +/opt/csghub/bin/csghub-dbm -a update -d "$VISIBILITY_DB_NAME" -N "$DB_USER" +{{- end }} + +# Execute SQL scripts +cd /opt/csghub/service/temporal && chpst -e "$ENV_DIR" /opt/csghub/embedded/etc/csghub/scripts/pg_executor + +: "${TEMPORAL_HOME:=/opt/csghub/embedded/etc/temporal}" export TEMPORAL_HOME -{{- $dataDir := (datasource "config").temporal.data | default "/var/opt/csghub/temporal" }} +{{- $dataDir := .temporal.data | default "/var/opt/csghub/temporal" }} cd {{ $dataDir }} # Setup temporal database -chpst -e {{ $dataDir }}/env -P \ - -u temporal:temporal \ +chpst -e /opt/csghub/service/temporal/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ "${TEMPORAL_HOME}"/auto-setup.sh -exec chpst -e {{ $dataDir }}/env -P \ - -u temporal:temporal \ +exec chpst -e /opt/csghub/service/temporal/env -P \ + -u {{ .run_user }}:{{ .run_user }} \ /opt/csghub/embedded/bin/temporal-server \ --allow-no-auth \ --env docker \ - --root {{ $dataDir }}/env \ - --config ../ \ - start \ No newline at end of file + --root /opt/csghub/service/temporal \ + --config etc \ + start diff --git a/opt/csghub/embedded/sv/temporal/templates/scripts/01_update_schema_version.sql.sample b/opt/csghub/embedded/sv/temporal/templates/scripts/01_update_schema_version.sql.sample index e9c27e31..f8adc6ea 100644 --- a/opt/csghub/embedded/sv/temporal/templates/scripts/01_update_schema_version.sql.sample +++ b/opt/csghub/embedded/sv/temporal/templates/scripts/01_update_schema_version.sql.sample @@ -26,8 +26,7 @@ SELECT pg_catalog.set_config('search_path', 'public', false); -- -- Type: DML; Schema: public; Owner: schema_version -- -{{- $temporal := (datasource "config").temporal }} -{{- $db := $temporal.postgresql }} +{{- $db := .temporal.postgresql }} BEGIN; LOCK TABLE public.schema_version IN EXCLUSIVE MODE; diff --git a/opt/csghub/embedded/sv/temporal_ui/templates/log/run.sample b/opt/csghub/embedded/sv/temporal_ui/templates/log/run.sample index 616a8a6b..f951cd16 100755 --- a/opt/csghub/embedded/sv/temporal_ui/templates/log/run.sample +++ b/opt/csghub/embedded/sv/temporal_ui/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/temporal_ui" }} -{{- if has (datasource "config").temporal_ui "log" }} -{{- if has (datasource "config").temporal_ui.log "dir" }} -{{- if (datasource "config").temporal_ui.log.dir }} -{{- $logDir = (datasource "config").temporal_ui.log.dir }} -{{- end }} +{{- if has .temporal_ui "log" }} +{{- if .temporal_ui.log.dir }} +{{- $logDir = .temporal_ui.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/temporal_ui/templates/run.sample b/opt/csghub/embedded/sv/temporal_ui/templates/run.sample index e266c3da..00ef633c 100755 --- a/opt/csghub/embedded/sv/temporal_ui/templates/run.sample +++ b/opt/csghub/embedded/sv/temporal_ui/templates/run.sample @@ -6,15 +6,10 @@ exec 2>&1 # fail on errors set -e -# Run the pre-start script to ensure that the service meets the startup conditions -if [ -x ./pre-start ]; then - ./pre-start -fi - -{{- $dataDir := (datasource "config").temporal_ui.data | default "/var/opt/csghub/temporal_ui" }} -{{- $temporalDataDir := (datasource "config").temporal.data | default "/var/opt/csghub/temporal" }} +{{- $dataDir := .temporal_ui.data | default "/var/opt/csghub/temporal_ui" }} +{{- $temporalDataDir := .temporal.data | default "/var/opt/csghub/temporal" }} cd {{ $dataDir }} exec chpst -P \ - -e /opt/csghub/etc/temporal_ui/env \ - -u temporal:temporal \ - /opt/csghub/embedded/bin/ui-server --env docker --root {{ $dataDir }}/env --config .. start + -e /opt/csghub/service/temporal_ui/env \ + -u {{ .run_user }}:{{ .run_user }} \ + /opt/csghub/embedded/bin/ui-server --env docker --root /opt/csghub/service/temporal_ui --config etc/ start diff --git a/opt/csghub/embedded/sv/temporal_worker/templates/log/run.sample b/opt/csghub/embedded/sv/temporal_worker/templates/log/run.sample index 444871d3..0ab93656 100755 --- a/opt/csghub/embedded/sv/temporal_worker/templates/log/run.sample +++ b/opt/csghub/embedded/sv/temporal_worker/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/temporal_worker" }} -{{- if has (datasource "config").temporal_worker "log" }} -{{- if has (datasource "config").temporal_worker.log "dir" }} -{{- if (datasource "config").temporal_worker.log.dir }} -{{- $logDir = (datasource "config").temporal_worker.log.dir }} -{{- end }} +{{- if has .temporal_worker "log" }} +{{- if .temporal_worker.log.dir }} +{{- $logDir = .temporal_worker.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/temporal_worker/templates/run.sample b/opt/csghub/embedded/sv/temporal_worker/templates/run.sample index 55a6fc5d..495a5a1b 100755 --- a/opt/csghub/embedded/sv/temporal_worker/templates/run.sample +++ b/opt/csghub/embedded/sv/temporal_worker/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("temporal-worker" "launch" "-l" {{ $server.log.level | quote }}) +args=("temporal-worker" "launch" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/sv/user/templates/log/run.sample b/opt/csghub/embedded/sv/user/templates/log/run.sample index 22a37f2a..a3316873 100755 --- a/opt/csghub/embedded/sv/user/templates/log/run.sample +++ b/opt/csghub/embedded/sv/user/templates/log/run.sample @@ -1,11 +1,9 @@ #!/bin/bash {{- $logDir := "/var/log/csghub/user" }} -{{- if has (datasource "config").user "log" }} -{{- if has (datasource "config").user.log "dir" }} -{{- if (datasource "config").user.log.dir }} -{{- $logDir = (datasource "config").user.log.dir }} -{{- end }} +{{- if has .user "log" }} +{{- if .user.log.dir }} +{{- $logDir = .user.log.dir }} {{- end }} {{- end }} diff --git a/opt/csghub/embedded/sv/user/templates/run.sample b/opt/csghub/embedded/sv/user/templates/run.sample index 9f4cc44c..2a2ccb23 100755 --- a/opt/csghub/embedded/sv/user/templates/run.sample +++ b/opt/csghub/embedded/sv/user/templates/run.sample @@ -6,18 +6,18 @@ exec 2>&1 # fail on errors set -e -{{- $server := (datasource "config").server }} -{{- $dataDir := $server.data | default "/var/opt/csghub/server" }} -{{- $configFile := printf "%s/config.toml" $dataDir }} +{{- $dataDir := .server.data | default "/var/opt/csghub/server" }} +{{- $configDir := "/opt/csghub/service/server/etc" }} +{{- $configFile := printf "%s/config.toml" $configDir }} # Define constants readonly SERVER_BIN="/opt/csghub/embedded/bin/csghub-server" -readonly ENV_DIR="/opt/csghub/etc/server/env" +readonly ENV_DIR="/opt/csghub/service/server/env" # Build command arguments -args=("user" "launch" "-l" {{ $server.log.level | quote }}) +args=("user" "launch" "-l" {{ .server.log.level | quote }}) [[ -f "{{ $configFile }}" ]] && args=("--config={{ $configFile }}" "${args[@]}") echo "Starting server..." cd {{ $dataDir }} -exec chpst -e "$ENV_DIR" -P -u root:root "$SERVER_BIN" "${args[@]}" +exec chpst -e "$ENV_DIR" -P -u {{ .run_user }}:{{ .run_user }} "$SERVER_BIN" "${args[@]}" diff --git a/opt/csghub/embedded/templates/config.tpl b/opt/csghub/embedded/templates/config.tpl deleted file mode 100644 index 37923fd8..00000000 --- a/opt/csghub/embedded/templates/config.tpl +++ /dev/null @@ -1,42 +0,0 @@ -{{- define "config.casdoor.conn" -}} - {{- $config := (datasource "config") -}} - {{- $csghub := $config.csghub }} - {{- $url := conv.URL $csghub.external_url -}} - {{- $host := (index (strings.Split ":" $url.Host) 0) | default "csghub.example.com" -}} - - {{- $casdoor := $config.casdoor }} - {{- $casdoorParts := strings.Split ":" $casdoor.listen -}} - {{- $casdoorHost := index $casdoorParts 0 | default "127.0.0.1" -}} - {{- $casdoorPort := index $casdoorParts 1 | default "8000" -}} - - {{- if eq $casdoorHost "127.0.0.1" -}} - {{- $casdoorHost = $host -}} - {{- end -}} - - {{- $result := coll.Dict "scheme" $url.Scheme "host" $casdoorHost "port" $casdoorPort -}} - {{- $result | data.ToYAML -}} -{{- end -}} - -{{- define "endpoint.casdoor" -}} - {{- $casdoor := tmpl.Exec "config.casdoor.conn" . | data.YAML -}} - {{- printf "%s://%s:%v" $casdoor.scheme $casdoor.host $casdoor.port -}} -{{- end -}} - -{{- define "config.casdoor.db" -}} - {{- $casdoor := (datasource "config").casdoor }} - {{- $db := $casdoor.postgresql -}} - {{- $config := coll.Dict - "dbname" ( $db.dbname | default "csghub_casdoor" ) - "host" ( $db.host | default "127.0.0.1" ) - "port" ( $db.port | default 5432 ) - "user" ( $db.user | default "csghub" ) - "password" ( $db.password | default (crypto.PBKDF2 ($db.user | default "csghub") "opencsg" 2048 8) ) - -}} - - {{- $config | data.ToYAML -}} -{{- end -}} - -{{- define "config.casdoor.db.dsn" -}} -{{- $config := tmpl.Exec "config.casdoor.db" . | data.YAML -}} -{{- printf "postgresql://%s:%s@%s:%v/%s?sslmode=disable" $config.user $config.password $config.host $config.port $config.dbname -}} -{{- end -}} \ No newline at end of file diff --git a/opt/csghub/embedded/templates/csghub.tpl b/opt/csghub/embedded/templates/csghub.tpl deleted file mode 100644 index 6206e871..00000000 --- a/opt/csghub/embedded/templates/csghub.tpl +++ /dev/null @@ -1,39 +0,0 @@ -{{- define "config.csghub" -}} - {{- $csghub := (datasource "config").csghub -}} - {{- $url := conv.URL $csghub.external_url -}} - {{- $urlParts := $url.Host | strings.Split ":" -}} - {{- $host := index $urlParts 0 | default "csghub.example.com" -}} - {{- $port := "" -}} - {{- if gt (len $urlParts) 1 -}} - {{- $port = index $urlParts 1 -}} - {{- else -}} - {{- if eq $url.Scheme "https" -}} - {{- $port = "443" -}} - {{- else -}} - {{- $port = "80" -}} - {{- end -}} - {{- end -}} - - {{- $result := coll.Dict - "external" $csghub.external_url - "scheme" $url.Scheme - "host" $host - "port" $port - -}} - {{- $result | data.ToYAML -}} -{{- end -}} - -{{- define "domain.root" -}} - {{- $rootDomain := "example.com" }} - {{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} - - {{- if and $csghub.host (regexp.Match `^[a-zA-Z0-9.-]+$` $csghub.host) }} - {{- $hostParts := strings.Split "." $csghub.host }} - {{- if le (len $hostParts) 2 }} - {{- $rootDomain = $csghub.host }} - {{- else }} - {{- $rootDomain = regexp.Replace "^[^.]+\\." "" $csghub.host }} - {{- end }} - {{- end }} - {{- $rootDomain -}} -{{- end -}} \ No newline at end of file diff --git a/opt/csghub/embedded/templates/seed.tpl b/opt/csghub/embedded/templates/seed.tpl deleted file mode 100644 index b4da5778..00000000 --- a/opt/csghub/embedded/templates/seed.tpl +++ /dev/null @@ -1,45 +0,0 @@ -# Generate or retrieve a persistent random seed used for cryptographic operations. -# The seed is stored in /etc/csghub/csghub.secret to ensure consistency across runs. -{{- define "GenSeed" -}} - {{- $seedFile := "/etc/csghub/csghub.secret" -}} - {{- $seed := "" -}} - {{- if file.Exists $seedFile -}} - {{- $seed = file.Read $seedFile -}} - {{- end -}} - {{- if not $seed -}} - {{- $seed = random.Alpha 48 -}} - {{- file.Write $seedFile $seed -}} - {{- end -}} - {{- $seed -}} -{{- end -}} - -# Generate a deterministic Hub API token based on a PBKDF2 hash. -# The token is derived using the static string "opencsg" as the password, -# the persistent seed as the salt, and a key length of 64 bytes. -{{- define "GenHubApiToken" -}} -{{- crypto.PBKDF2 "opencsg" (tmpl.Exec "GenSeed" .) 1024 64 -}} -{{- end }} - -# Generate a deterministic Client ID using the seed and input context. -# The ID is a SHA256 hash truncated to 20 characters. -{{- define "GenClientId" -}} - {{- printf "%s%s" (tmpl.Exec "GenSeed" .) . | crypto.SHA256 | strings.Trunc 20 -}} -{{- end -}} - -# Generate a deterministic Client Secret using the seed and input context. -# The secret is derived from the SHA256 hash with the first 24 characters removed. -{{- define "GenClientSecret" -}} - {{- printf "%s%s" (tmpl.Exec "GenSeed" .) . | crypto.SHA256 | regexp.Replace "^.{24}" "" -}} -{{- end -}} - -# Generate a base64-looking password -{{- define "GenInitPass" -}} - {{- /* Generate a deterministic 24-character password from seed and input */ -}} - {{- $seed := tmpl.Exec "GenSeed" . -}} - {{- $raw := printf "%s%s" $seed . | base64.Encode | strings.Trunc 24 -}} - {{- $chars := strings.Split $raw "" -}} - {{- $reversed := "" -}} - {{- range $i, $c := $chars -}} - {{- $reversed = printf "%s%s" $c $reversed -}} - {{- end -}} -{{- end -}} \ No newline at end of file diff --git a/opt/csghub/etc/gitaly/config.toml.sample b/opt/csghub/etc/gitaly/config.toml.sample deleted file mode 100755 index 2fcc438c..00000000 --- a/opt/csghub/etc/gitaly/config.toml.sample +++ /dev/null @@ -1,153 +0,0 @@ -# Example Gitaly configuration file with all configuration blocks dynamically defined. - -{{- $gitaly := (datasource "config").gitaly }} -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML }} -{{- $server := (datasource "config").server }} -{{- $dataDir := $gitaly.data_dir | default "/var/opt/csghub/gitaly" }} -{{- $listenAddr := $gitaly.listen_addr | default "" -}} -{{- if eq $listenAddr "" }} -# Unix socket path for Gitaly -socket_path = {{ $gitaly.socket_path | default (printf "%s/gitaly.socket" $dataDir) | quote }} -{{- end }} - -# Directory containing Gitaly executables -bin_dir = {{ $gitaly.bin_dir | quote }} - -# Runtime directory for Gitaly -runtime_dir = {{ printf "%s/run" $dataDir | quote }} - -# Unencrypted TCP listen address -listen_addr = {{ $gitaly.listen_addr | quote }} - -{{- if and $gitaly.tls_listen_addr (not $gitaly.listen_addr) }} -# TCP with TLS listen address -tls_listen_addr = {{ $gitaly.tls_listen_addr | quote }} -{{- end }} - -# Prometheus metrics listen address -prometheus_listen_addr = {{ $gitaly.prometheus_listen_addr | quote }} - -[auth] -# Auth token required for Gitaly authentication -token = {{ $gitaly.auth.token | default (crypto.PBKDF2 "gitaly" "opencsg" 2048 8) | quote }} -transitioning = {{ $gitaly.auth.transitioning }} - -{{- if has $gitaly "tls" }} -# Gitaly supports TLS encryption. You must bring your own certificates because this isn’t provided automatically. -[tls] -{{- if $gitaly.tls.certificate_path }} -certificate_path = {{ $gitaly.tls.certificate_path | default "/var/opt/csghub/gitaly/ssl/cert.cert" | quote }} -{{- end }} -{{- if $gitaly.tls.key_path }} -key_path = {{ $gitaly.tls.key_path | default "/var/opt/csghub/gitaly/ssl/key.pem" | quote }} -{{- end }} -{{ end }} - -# Git settings -[git] -use_bundled_binaries = {{ $gitaly.git.use_bundled_binaries }} -ignore_gitconfig = {{ $gitaly.git.ignore_gitconfig }} -bin_path = {{ $gitaly.git.bin_path | quote }} -catfile_cache_size = {{ $gitaly.git.catfile_cache_size }} -signing_key = "{{ $gitaly.git.signing_key }}" - - # Git configuration - {{- range $config := $gitaly.git.config }} - [[git.config]] - key = {{ $config.key | quote }} - value = {{ $config.value | quote }} - {{ end }} - -# Storages configuration -{{- range $gitaly.storage }} -[[storage]] -name = {{ .name | quote }} -path = {{ .path | default (printf "%s/repositories" $dataDir) | quote }} -{{- end }} - -[logging] -format = {{ $gitaly.logging.format | quote }} -level = {{ $gitaly.logging.level | quote }} -dir = {{ $gitaly.logging.dir | quote }} -sentry_dsn = {{ $gitaly.logging.sentry_dsn | quote }} -sentry_environment = {{ $gitaly.logging.sentry_environment | quote }} - -[prometheus] -grpc_latency_buckets = [{{- range $i, $v := $gitaly.prometheus.grpc_latency_buckets }}{{if $i}}, {{end}}{{$v}}{{- end }}] - -[hooks] -custom_hooks_dir = {{ $gitaly.hooks.custom_hooks_dir | default (printf "%s/custom_hooks" $dataDir) | quote }} - -[gitlab] -url = {{ $csghub.external | quote }} -relative_url_root = {{ $csghub.relative_url_root | default "/" | quote }} -{{- if $gitaly.secret_file }} -secret_file = {{ $gitaly.secret_file | quote }} -{{- else }} -secret = {{ $gitaly.secret | default (tmpl.Exec "GenSeed" . | crypto.SHA256) | quote }} -{{- end }} - -[gitlab.http-settings] -read_timeout = {{ $gitaly.http_settings.read_timeout }} -self_signed_cert = {{ $gitaly.http_settings.self_signed_cert }} - -{{ range $concurrency := $gitaly.concurrency }} -[[concurrency]] -{{- range $key, $value := $concurrency }} -{{- if test.IsKind "string" $value }} -{{ $key }} = {{ $value | quote }} -{{- else if test.IsKind "number" $value }} -{{ $key }} = {{ $value }} -{{- end }} -{{- end }} -{{ end }} - -{{- range $limiting := $gitaly.rate_limiting }} -[[rate_limiting]] -{{- range $key, $value := $limiting }} -{{- if test.IsKind "string" $value }} -{{ $key }} = {{ $value | quote }} -{{- else if test.IsKind "number" $value }} -{{ $key }} = {{ $value }} -{{- end }} -{{- end }} -{{ end }} - -[daily_maintenance] -disabled = {{ $gitaly.daily_maintenance.disable }} -start_hour = {{ $gitaly.daily_maintenance.start_hour }} -start_minute = {{ $gitaly.daily_maintenance.start_minute }} -duration = {{ $gitaly.daily_maintenance.duration | quote }} -storages = [{{- range $i, $v := $gitaly.daily_maintenance.storages }}{{if $i}}, {{end}}{{ $v | quote }}{{- end }}] - -{{- if has $gitaly "cgroups" }} -[cgroups] -mountpoint = {{ $gitaly.cgroups.mountpoint | default "/sys/fs/cgroup" | quote }} -hierarchy_root = {{ $gitaly.cgroups.hierarchy_root | default "gitaly" | quote }} -memory_bytes = {{ $gitaly.cgroups.memory_bytes | default 64424509440 }} -cpu_shares = {{ $gitaly.cgroups.cpu_shares | default 1024 }} -cpu_quota_us = {{ $gitaly.cgroups.cpu_quota_us | default 400000 }} -[cgroups.repositories] -count = {{ $gitaly.cgroups.repositories.count | default 500 }} -memory_bytes = {{ $gitaly.cgroups.repositories.memory_bytes | default 12884901888 }} -cpu_shares = {{ $gitaly.cgroups.repositories.cpu_shares | default 512 }} -cpu_quota_us = {{ $gitaly.cgroups.repositories.cpu_quota_us | default 200000 }} -{{ end }} - -{{- if has $gitaly "backup" }} -[backup] -go_cloud_url = {{ $gitaly.backup.go_cloud_url | default "gs://gitaly-backups" | quote }} -layout = {{ $gitaly.backup.layout | default "pointer" | quote }} -wal_backup_go_cloud_url = {{ $gitaly.backup.wal_backup_go_cloud_url | default "gs://gitaly-wal-backups" | quote }} -wal_backup_worker_count = {{ $gitaly.backup.wal_backup_worker_count | default 1 }} -buffer_size = {{ $gitaly.backup.buffer_size | default 0 }} -{{ end }} - -{{- if has $gitaly "bundle_uri" }} -[bundle_uri] -go_cloud_url = {{ $gitaly.bundle_uri.go_cloud_url | default "gs://my-bundle-uri-bucket" | quote }} -{{ end }} - -[timeout] -upload_pack_negotiation = {{ $gitaly.timeout.upload_pack_negotiation | default "10m" | quote }} -upload_archive_negotiation = {{ $gitaly.timeout.upload_archive_negotiation | default "1m" | quote }} diff --git a/opt/csghub/etc/prometheus/prometheus.yml.sample b/opt/csghub/etc/prometheus/prometheus.yml.sample deleted file mode 100644 index 02d9d396..00000000 --- a/opt/csghub/etc/prometheus/prometheus.yml.sample +++ /dev/null @@ -1,23 +0,0 @@ -{{- $prometheus := (datasource "config").prometheus }} -global: - scrape_interval: {{ $prometheus.scrape_interval | default "15s" }} - evaluation_interval: {{ $prometheus.evaluation_interval | default "15s" }} - scrape_timeout: {{ $prometheus.scrape_timeout | default "10s" }} - -{{- if $prometheus.external_labels }} - external_labels: - {{- range $key, $value := $prometheus.external_labels }} - {{ $key }}: {{ $value | quote }} - {{- end }} -{{- end }} - -rule_files: - # - "alert.rules.yml" - -scrape_configs: - - job_name: "prometheus" - static_configs: - - targets: ["localhost:9090"] -{{- if $prometheus.scrape_configs }} -{{ toYAML $prometheus.scrape_configs | strings.Indent 2 }} -{{- end }} \ No newline at end of file diff --git a/opt/csghub/etc/server/config.toml.sample b/opt/csghub/etc/server/config.toml.sample deleted file mode 100644 index c15d6e93..00000000 --- a/opt/csghub/etc/server/config.toml.sample +++ /dev/null @@ -1,301 +0,0 @@ -{{- $csghub := tmpl.Exec "config.csghub" . | data.YAML -}} -{{- $server := (datasource "config").server -}} -{{- $runner := (datasource "config").runner -}} -{{- $praefect := (datasource "config").praefect -}} -{{- $gitlabShell := (datasource "config").gitlab_shell -}} - -saas = {{ $server.saas }} -oversea = false -instance_id = "" -enable_swagger = {{ $server.enable_swagger }} -enable_https = {{ $server.enable_https }} -api_token = {{ tmpl.Exec "GenHubApiToken" . | quote }} -docs_host = {{ printf "%s://%s:6636" $csghub.scheme $csghub.host | quote }} -server_failure_redirect_url = {{ printf "%s/errors/server-error" $csghub.external | quote }} - -[api_server] -port = {{ $server.listen_port }} -public_domain = {{ $csghub.external | quote }} -{{- $shell_port := $gitlabShell.ssh_port }} -{{- if eq $shell_port 22 }} -ssh_domain = {{ printf "git@%s:%d" (conv.ToString $csghub.host) (conv.ToInt64 $shell_port) | quote }} -{{- else }} -ssh_domain = {{ printf "ssh://git@%s:%d" (conv.ToString $csghub.host) (conv.ToInt64 $shell_port) | quote }} -{{- end }} - -{{ $mirrorRepo := (datasource "config").mirror_repo }} -{{- $mirrorLfs := (datasource "config").mirror_lfs -}} -[mirror] -url = {{ printf "http://localhost:%d" $mirrorRepo.listen_port | quote }} -token = "" -port = {{ $mirrorRepo.listen_port }} -{{- if $server.saas }} -remote = true -{{- else }} -remote = false -{{- end }} -session_secret_key = {{ tmpl.Exec "GenSeed" . | crypto.SHA1 | quote }} -worker_number = {{ $mirrorRepo.worker_number }} -lfs_concurrency = {{ $mirrorLfs.lfs_concurrency }} -part_size = {{ $mirrorLfs.part_size }} - -{{ $postgresql := $server.postgresql }} -{{- $pgUser := $postgresql.user -}} -{{- $pgPassword := $postgresql.password | default (crypto.PBKDF2 $pgUser "opencsg" 2048 8) -}} -[database] -driver = "pg" -dsn = {{ printf "postgresql://%s:%s@%s:%d/%s?sslmode=disable" (conv.ToString $pgUser) (conv.ToString $pgPassword) (conv.ToString $postgresql.host) (conv.ToInt64 $postgresql.port) (conv.ToString $postgresql.name) | quote }} -timezone = {{ $postgresql.timezone | quote }} - -{{ $redis := $server.redis }} -{{- $redisUser := $redis.auth.user -}} -{{- $redisPassword := $redis.auth.password | default (crypto.PBKDF2 $redisUser "opencsg" 2048 8) -}} -[redis] -endpoint = {{ $redis.endpoint | quote }} -max_retries = {{ $redis.max_retries }} -min_idle_connections = {{ $redis.min_idle_connections }} -user = {{ $redisUser | quote }} -password = {{ $redisPassword | quote }} -sentinel_mode = {{ $redis.sentinel.mode }} -sentinel_endpoint = {{ $redis.sentinel.endpoint | quote }} -sentinel_master = {{ $redis.sentinel.master | quote }} - -[git_server] -type = "gitaly" - -{{ $gitaly := $server.gitaly }} -{{- $gitalyToken := $gitaly.token | default (crypto.PBKDF2 "gitaly" "opencsg" 2048 8) -}} -[gitaly_server] -address = {{ $gitaly.address | quote }} -storge = {{ $gitaly.storage | quote }} -token = {{ $gitalyToken | quote }} -jwt_secret ={{ $gitaly.secret | default (tmpl.Exec "GenSeed" . | crypto.SHA256) | quote }} - -[frontend] -url = {{ $csghub.external | quote }} - -{{ $s3 := $server.s3 }} -{{- $accessKeyId := $s3.access_key -}} -{{- $secretKey := $s3.secret_key | default (crypto.PBKDF2 $accessKeyId "opencsg" 2048 8) -}} -[s3] -access_key_id = {{ $accessKeyId | quote }} -access_key_secret = {{ $secretKey | quote }} -region = {{ $s3.region | quote }} -endpoint = {{ $s3.endpoint | default (printf "%s:9000" $csghub.host) | quote }} -internal_endpoint = {{ $s3.internal_endpoint | quote }} -bucket = {{ $s3.bucket | quote }} -enable_ssl = {{ $s3.secure }} -{{- if $s3.path_style }} -bucket_lookup = "path" -{{- else }} -bucket_lookup = "auto" -{{- end }} - -[jwt] -signing_key = {{ $gitaly.secret | default (tmpl.Exec "GenSeed" . | crypto.SHA256) | quote }} -valid_hour = 24 - -[space] -internal_root_domain = "spaces.app.internal:8083" -{{- if $runner.use_public_domain }} -public_root_domain = {{ printf "%s:%v" $csghub.host $csghub.port | quote }} -{{- else }} -public_root_domain = "" -{{- end }} -session_secret_key = {{ tmpl.Exec "GenSeed" . | crypto.SHA1 | quote }} -pypi_index_url = {{ $runner.pip_index_url | quote }} - -{{ $model := $runner.model -}} -[model] -deploy_timeout_in_min = {{ $model.deploy_timeout }} -download_endpoint = {{ $csghub.external | quote }} -docker_reg_base = {{ $model.registry | quote }} -nim_docker_secret_name = "ngc-secret" -nim_ngc_secret_name = "nvidia-nim-secrets" - -[event] -sync_interval = {{ $mirrorRepo.sync_interval }} - -{{ $casdoor := (datasource "config").casdoor -}} -{{- $serverCasdoor := $server.casdoor -}} -[casdoor] -client_id = {{ tmpl.Exec "GenClientId" "CSGHub" | quote }} -client_secret = {{ tmpl.Exec "GenClientSecret" "CSGHub" | quote }} -endpoint = {{ $serverCasdoor.endpoint | default (tmpl.Exec "endpoint.casdoor" .) | quote }} -certificate = {{ $serverCasdoor.certificate | quote }} -organization_name = {{ $serverCasdoor.organization_name | quote }} -application_name = {{ $serverCasdoor.application_name | quote }} - -{{ $nats := (datasource "config").nats }} -{{- $natsUser := $nats.auth.user -}} -{{- $natsPassword := $nats.auth.password | default (crypto.PBKDF2 $natsUser "opencsg" 2048 8) -}} -{{- $serverNats := $server.nats -}} -[nats] -url = {{ $serverNats.url | default (printf "nats://%s:%s@localhost:4222" (conv.ToString $natsUser) (conv.ToString $natsPassword)) | quote }} -msg_fetch_timeout_in_sec = {{ $serverNats.msg_fetch_timeout_in_sec }} - -{{ $accounting := (datasource "config").accounting }} -{{- $actParts := $accounting.listen | strings.Split ":" -}} -{{- $actHost := index $actParts 0 -}} -{{- $actPort := "" -}} -{{- if gt (len $actParts) 1 -}} - {{- $actPort = index $actParts 1 -}} -{{- else -}} - {{- $actPort = "8086" -}} -{{- end -}} -[accounting] -host = {{ printf "http://%s" $actHost | quote }} -port = {{ $actPort }} - -{{ $user := (datasource "config").user }} -{{- $userParts := $user.listen | strings.Split ":" -}} -{{- $userHost := index $userParts 0 -}} -{{- $userPort := "" -}} -{{- if gt (len $userParts) 1 -}} - {{- $userPort = index $userParts 1 -}} -{{- else -}} - {{- $actPort = "8088" -}} -{{- end -}} -[user] -host = {{ printf "http://%s" $userHost | quote }} -port = {{ $userPort }} -signin_success_redirect_url = {{ printf "%s/server/callback" $csghub.external | quote }} - -{{ $multi_sync := $server.multi_sync -}} -[multi_sync] -enabled = {{ $multi_sync.enabled }} -saas_api_domain = "https://hub.opencsg.com" -saas_sync_domain = "https://sync.opencsg.com" - -{{ $telemetry := $server.telemetry -}} -[telemetry] -enable = {{ $telemetry.enabled | default true }} -report_url = "http://hub.opencsg.com/api/v1/telemetry" - -[auto_clean] -instance = false - -{{ $dataset := $server.dataset -}} -[dataset] -prompt_max_jsonl_file_size = {{ $dataset.prompt_max_jsonl_file_size }} - -{{ $dataflow := $server.dataflow -}} -{{- $dfUrl := conv.URL $dataflow.address -}} -{{- $dfUrlParts := $dfUrl.Host | strings.Split ":" -}} -{{- $dfHost := index $dfUrlParts 0 -}} -{{- $dfPort := "" -}} -{{- if gt (len $dfUrlParts) 1 -}} - {{- $dfPort = index $dfUrlParts 1 -}} -{{- else -}} - {{- if eq $dfUrl.Scheme "https" -}} - {{- $dfPort = "443" -}} - {{- else -}} - {{- $dfPort = "80" -}} - {{- end -}} -{{- end -}} -[dataflow] -host = {{ printf "%s://%s" $dfUrl.Scheme $dfHost | quote }} -port = {{ $dfPort }} - -{{ $moderation := (datasource "config").moderation -}} -{{- if $moderation.enable -}} -{{- $moderationAddress := $moderation.address -}} -{{- $moderationUrl := conv.URL $moderationAddress -}} -{{- $moderationParts := $moderationUrl.Host | strings.Split ":" -}} -{{- $moderationHost := index $moderationParts 0 | default "127.0.0.1" -}} -{{- $moderationPort := "" -}} -{{- if gt (len $moderationParts) 1 }} - {{- $moderationPort = index $moderationParts 1 -}} -{{- else -}} - {{- $moderationPort = "8089" -}} -{{- end -}} -[moderation] -host = {{ printf "%s://%s" $moderationUrl.Scheme $moderationHost | quote }} -port = {{ $moderationPort }} -encoded_sensitive_words = {{ $moderation.encoded_sensitive_words | quote }} - -{{ $sensitive := $moderation.sensitive_check -}} -[sensitive_check] -enable = true -access_key_id = {{ $sensitive.access_key_id | quote }} -access_key_secret = {{ $sensitive.access_key_secret | quote }} -region = {{ $sensitive.region | quote }} -endpoint = {{ $sensitive.endpoint | quote }} -enable_ssl = {{ $sensitive.enable_ssl }} -{{- end -}} - -{{ $workflow := $server.workflow -}} -[workflow] -endpoint = {{ $workflow.endpoint | quote }} - -[cron_job] -sync_as_client_cron_expression = {{ $multi_sync.sync_as_client_cron_expression | quote }} -calc_recom_score_cron_expression = {{ $multi_sync.calc_recom_score_cron_expression | quote }} - -{{ $proxy := $server.proxy -}} -[proxy] -hosts = [{{- range $k, $v := $proxy.hosts }}{{if $k}}, {{end}}{{$v | quote }}{{- end }}] - -{{ $instrumentation := $server.instrumentation -}} -[instrumentation] -otlp_logging = {{ $instrumentation.otlp_logging }} -otlp_endpoint = {{ $instrumentation.otlp_endpoint | quote }} - -{{ $git := $server.git -}} -[git] -operation_timeout = {{ $git.operation_timeout }} -{{- if $praefect.enable }} -check_file_size_enabled = false -{{- else }} -check_file_size_enabled = {{ $git.check_file_size_enabled }} -{{- end }} -max_un_lfs_file_size = {{ $git.max_un_lfs_file_size }} -skip_lfs_file_validation = {{ $git.skip_lfs_file_validation }} - - -{{ $notifier := (datasource "config").notifier -}} -{{- if $notifier.enable }} -{{- $notifyListen := $notifier.listen -}} -{{- $notifyParts := $notifyListen | strings.Split ":" -}} -{{- $notifyHost := index $notifyParts 0 | default "127.0.0.1" -}} -{{- $notifyPort := "8095" -}} -{{- if eq (len $notifyParts) 2 -}} - {{- $notifyPort = index $notifyParts 1 -}} -{{- end -}} -[notification] -host = {{ printf "http://%s" $notifyHost | quote }} -port = {{ $notifyPort }} -mailer_host = {{ $notifier.smtp.host | quote }} -mailer_port = {{ $notifier.smtp.port }} -mailer_username = {{ $notifier.smtp.username | quote }} -mailer_password = {{ $notifier.smtp.password | quote }} -repo_sync_timezone = {{ $notifier.repo_sync_timezone | quote }} -broadcast_user_page_size = {{ $notifier.broadcast_user_page_size }} -broadcast_email_page_size = {{ $notifier.broadcast_email_page_size }} - -{{ $feishu := $notifier.feishu -}} -[feishu] -app_id = {{ $feishu.app_id | quote }} -app_secret = {{ $feishu.app_secret | quote }} -batch_send_message_cron_expression = {{ $feishu.batch_send_message_cron_expression | quote }} -max_request_content_size = {{ $feishu.max_request_content_size }} -max_delay_duration = {{ $feishu.max_delay_duration }} -chat_ids_cache_ttl = {{ $feishu.chat_ids_cache_ttl }} -{{- end }} - -{{ $prometheus := $server.prometheus -}} -[prometheus] -api_address = {{ printf "%s/api/v1/query" $prometheus.address | quote }} - -{{ $logcollector := $server.logcollector -}} -[logcollector] -loki_url = {{ $logcollector.loki_url | quote }} - -{{ $aigateway := (datasource "config").aigateway -}} -[aigateway] -port = {{ $aigateway.port }} -moderation_bypass_sensitive_check = {{ $aigateway.moderation_bypass_sensitive_check }} -advertise_addr = "{{ $csghub.host }}:8094" - -[storagegateway] -enable_presigned_url_proxy = {{ $server.enable_presigned_url_proxy }} \ No newline at end of file diff --git a/packages/debian-based/DEBIAN/postinst b/packages/debian-based/DEBIAN/postinst index 1e2c002f..5043a868 100755 --- a/packages/debian-based/DEBIAN/postinst +++ b/packages/debian-based/DEBIAN/postinst @@ -81,7 +81,7 @@ case "$1" in if [ -n "$2" ]; then if [ ! -e /etc/csghub/csghub.yaml ]; then mkdir -p /etc/csghub || true - cp /opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample /etc/csghub/csghub.yaml + cp /opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample /etc/csghub/csghub.yaml fi fi @@ -90,11 +90,11 @@ case "$1" in create_symlinks "/opt/csghub/embedded/python/bin" "/opt/csghub/embedded/bin" echo "Copy systemd unit files." - cp -f /opt/csghub/etc/csghub/templates/system/csghub-runsvdir.service /etc/systemd/system/ + cp -f /opt/csghub/embedded/etc/csghub/csghub-runsvdir.service /etc/systemd/system/ echo "Copy configuration files." if [ ! -f "/etc/csghub/csghub.yaml" ]; then - cp -rf /opt/csghub/etc/csghub/templates/csghub /etc/csghub + cp -rf /opt/csghub/embedded/etc/csghub/templates /etc/csghub mv /etc/csghub/csghub.yaml.sample /etc/csghub/csghub.yaml fi diff --git a/packages/rhel-based/omnibus-csghub.spec b/packages/rhel-based/omnibus-csghub.spec index 68f64506..d1d5b390 100644 --- a/packages/rhel-based/omnibus-csghub.spec +++ b/packages/rhel-based/omnibus-csghub.spec @@ -33,7 +33,7 @@ mkdir -p %{buildroot}/etc/csghub mkdir -p %{buildroot}/opt/csghub cp -a %{_builddir}/opt/csghub/* %{buildroot}/opt/csghub/ -cp -a %{_builddir}/opt/csghub/etc/csghub/templates/system/csghub-runsvdir.service %{buildroot}/etc/systemd/system/ +cp -a %{_builddir}/opt/csghub/embedded/etc/csghub/csghub-runsvdir.service %{buildroot}/etc/systemd/system/ %pre #!/bin/sh @@ -52,7 +52,7 @@ done # Create central configuration file if [ ! -e /etc/csghub/csghub.yaml ]; then mkdir -p /etc/csghub || true - cp /opt/csghub/etc/csghub/templates/csghub/csghub.yaml.sample /etc/csghub/csghub.yaml + cp /opt/csghub/embedded/etc/csghub/templates/csghub.yaml.sample /etc/csghub/csghub.yaml fi # Enable and start service diff --git a/toolbox/csghub-ctl b/toolbox/csghub-ctl index 0196c2a7..349b8c4d 160000 --- a/toolbox/csghub-ctl +++ b/toolbox/csghub-ctl @@ -1 +1 @@ -Subproject commit 0196c2a7570b7fe042c8c116cd16f5a59834674a +Subproject commit 349b8c4d0776b4f5689d06bb4f5764d5db2d2be8