From d4658ec3f205c94342c6ddb5e4f8c91a498913f7 Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Mon, 31 Mar 2025 21:03:00 -0400 Subject: [PATCH 1/2] add scaffold-gs documentation --- docs/nerfology/methods/index.md | 1 + docs/nerfology/methods/scaffold_gs.md | 38 ++++++++++++++++++++++++++ nerfstudio/configs/external_methods.py | 15 ++++++++++ 3 files changed, 54 insertions(+) create mode 100644 docs/nerfology/methods/scaffold_gs.md diff --git a/docs/nerfology/methods/index.md b/docs/nerfology/methods/index.md index 320d6ae97f..1e776976a4 100644 --- a/docs/nerfology/methods/index.md +++ b/docs/nerfology/methods/index.md @@ -49,6 +49,7 @@ The following methods are supported in nerfstudio: BioNeRF NeRFtoGSandBack OpenNeRF + Scaffold-GS ``` (own_method_docs)= diff --git a/docs/nerfology/methods/scaffold_gs.md b/docs/nerfology/methods/scaffold_gs.md new file mode 100644 index 0000000000..211441a69c --- /dev/null +++ b/docs/nerfology/methods/scaffold_gs.md @@ -0,0 +1,38 @@ +# Scaffold-GS + +

Unofficial implementation of "Scaffold-GS: Structured 3D Gaussians for View-Adaptive Rendering"

+ +```{button-link} https://city-super.github.io/scaffold-gs/ +:color: primary +:outline: +Paper Website +``` +```{button-link} https://github.com/brian-xu/scaffold-gs-nerfstudio +:color: primary +:outline: +Code +``` + + + + +### Installation +Ensure that nerfstudio has been installed according to the instructions. Then run the following command: +``` +pip install git+https://github.com/brian-xu/scaffold-gs-nerfstudio +``` +You must also install the correct torch_scatter for your environment (https://pytorch-geometric.com/whl/torch-2.1.2%2Bcu118.html). + + +### Running Model + +This repository creates a new Nerfstudio method named "scaffold-gs". To train with it, run the command: + +```bash +ns-train scaffold-gs --data [PATH] +``` + +## Overview +Scaffold-GS replaces the Gaussian kernel described in [3D Gaussian Splatting](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/) with neural Gaussians that are bound to anchor points. During rasterization, the attributes (opacity, color, scale, rotation) of these neural Gaussians are calculated with respect to the viewing direction and distance. This view-adaptive rendering improves results in challenging conditions, such as texture-less areas, insufficient observations, fine-scale details, view-dependent light effects and multi-scale observations. \ No newline at end of file diff --git a/nerfstudio/configs/external_methods.py b/nerfstudio/configs/external_methods.py index 002b3299b6..8fc206601a 100644 --- a/nerfstudio/configs/external_methods.py +++ b/nerfstudio/configs/external_methods.py @@ -298,6 +298,21 @@ class ExternalMethod: ) ) +# Scaffold-GS +external_methods.append( + ExternalMethod( + """[bold yellow]Scaffold-GS[/bold yellow] + For more information visit: https://docs.nerf.studio/nerfology/methods/scaffold_gs.html + + To enable Scaffold-GS, you must install it first by running: + [grey]pip install git+https://github.com/brian-xu/scaffold-gs-nerfstudio"[/grey]""", + configurations=[ + ("scaffold-gs", "Scaffold-GS: Structured 3D Gaussians for View-Adaptive Rendering"), + ], + pip_package="git+https://github.com/brian-xu/scaffold-gs-nerfstudio", + ) +) + @dataclass class ExternalMethodDummyTrainerConfig: From 5dbb321bb5b584135324574376e7bd9fed409e5f Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Thu, 1 May 2025 20:14:37 -0400 Subject: [PATCH 2/2] add gsdf to methods --- docs/nerfology/methods/index.md | 1 + docs/nerfology/methods/scaffold_gs.md | 20 +++++++++++++++----- nerfstudio/configs/external_methods.py | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/nerfology/methods/index.md b/docs/nerfology/methods/index.md index 1e776976a4..c4d02bf071 100644 --- a/docs/nerfology/methods/index.md +++ b/docs/nerfology/methods/index.md @@ -50,6 +50,7 @@ The following methods are supported in nerfstudio: NeRFtoGSandBack OpenNeRF Scaffold-GS + GSDF ``` (own_method_docs)= diff --git a/docs/nerfology/methods/scaffold_gs.md b/docs/nerfology/methods/scaffold_gs.md index 211441a69c..b0edf6be50 100644 --- a/docs/nerfology/methods/scaffold_gs.md +++ b/docs/nerfology/methods/scaffold_gs.md @@ -1,11 +1,16 @@ -# Scaffold-GS +# Scaffold-GS + GSDF -

Unofficial implementation of "Scaffold-GS: Structured 3D Gaussians for View-Adaptive Rendering"

+

Unofficial implementations of "Scaffold-GS: Structured 3D Gaussians for View-Adaptive Rendering" and "GSDF: 3DGS Meets SDF for Improved Neural Rendering and Reconstruction"

```{button-link} https://city-super.github.io/scaffold-gs/ :color: primary :outline: -Paper Website +Scaffold-GS Website +``` +```{button-link} https://city-super.github.io/GSDF/ +:color: primary +:outline: +GSDF Website ``` ```{button-link} https://github.com/brian-xu/scaffold-gs-nerfstudio :color: primary @@ -28,11 +33,16 @@ You must also install the correct torch_scatter for your environment (https://py ### Running Model -This repository creates a new Nerfstudio method named "scaffold-gs". To train with it, run the command: +This repository creates two new Nerfstudio methods, named "scaffold-gs" and "gsdf". To train with them, run the following commands: ```bash ns-train scaffold-gs --data [PATH] +ns-train gsdf --data [PATH] ``` +GSDF also supports mesh export. Follow the guide for [exporting geometry](../../quickstart/export_geometry.md). + ## Overview -Scaffold-GS replaces the Gaussian kernel described in [3D Gaussian Splatting](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/) with neural Gaussians that are bound to anchor points. During rasterization, the attributes (opacity, color, scale, rotation) of these neural Gaussians are calculated with respect to the viewing direction and distance. This view-adaptive rendering improves results in challenging conditions, such as texture-less areas, insufficient observations, fine-scale details, view-dependent light effects and multi-scale observations. \ No newline at end of file +Scaffold-GS replaces the Gaussian kernel described in [3D Gaussian Splatting](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/) with neural Gaussians that are bound to anchor points. During rasterization, the attributes (opacity, color, scale, rotation) of these neural Gaussians are calculated with respect to the viewing direction and distance. This view-adaptive rendering improves results in challenging conditions, such as texture-less areas, insufficient observations, fine-scale details, view-dependent light effects and multi-scale observations. + +GSDF is a dual-branch model that uses 3D Gaussian Splatting to supervise the training of a neural Signed Distance Field. The gaussian splatting branch provides depth maps to supervise SDF training. Once sufficiently initialized, the SDF can be used to guide the splitting and pruning of gaussians, ensuring geometric consistency and reducing floaters. \ No newline at end of file diff --git a/nerfstudio/configs/external_methods.py b/nerfstudio/configs/external_methods.py index 8fc206601a..66d1046f9a 100644 --- a/nerfstudio/configs/external_methods.py +++ b/nerfstudio/configs/external_methods.py @@ -313,6 +313,21 @@ class ExternalMethod: ) ) +# GSDF +external_methods.append( + ExternalMethod( + """[bold yellow]GSDF/bold yellow] + For more information visit: https://docs.nerf.studio/nerfology/methods/scaffold_gs.html + + To enable GSDF, you must install it first by running: + [grey]pip install git+https://github.com/brian-xu/scaffold-gs-nerfstudio"[/grey]""", + configurations=[ + ("gsdf", "Scaffold-GS: GSDF: 3DGS Meets SDF for Improved Neural Rendering and Reconstruction"), + ], + pip_package="git+https://github.com/brian-xu/scaffold-gs-nerfstudio", + ) +) + @dataclass class ExternalMethodDummyTrainerConfig: