Merge pull request #29 from THD-Spatial/27-bugfix-docker-setup

Fix Docker Setup & Improve Error Handling

Author: jravani

README.md

@@ -16,8 +16,7 @@ City2TABULA focuses on scalable, database-centric processing of large national-
 
 ## Getting started
 
-- Setup & installation guide (Docker + manual): [docs/getting-started/setup.md](docs/installation/setup.md)
-- Full documentation: [city2tabula docs](https://thd-spatial.github.io/city2tabula)
+For setup and installation guide, please refer to [docs/getting-started/setup.md](https://thd-spatial.github.io/city2tabula/installation/setup/)
 
 Local docs preview (MkDocs):
 

cmd/main.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/THD-Spatial/City2TABULA/internal/config"
@@ -63,6 +62,14 @@ func main() {
 	if *createDB {
 		utils.Info.Println("Creating complete City2TABULA database...")
 		if err := db.CreateCompleteDatabase(&config, pool); err != nil {
+			if strings.Contains(err.Error(), "already exists") {
+				utils.Error.Println("Failed to create database:", err)
+				utils.Info.Println("")
+				utils.Info.Println("The database appears to have been set up already.")
+				utils.Info.Println("To start fresh, run:  ./city2tabula --reset-all  (or 'make reset-db')")
+				utils.Info.Println("To re-extract only:   ./city2tabula -extract-features  (or 'make extract-features')")
+				os.Exit(1)
+			}
 			utils.Error.Fatalf("Failed to create database: %v", err)
 		}
 		utils.Info.Println("Database creation completed successfully")
@@ -132,8 +139,7 @@ func runFeatureExtraction(config *config.Config, pool *pgxpool.Pool) error {
 	}
 
 	// Check if there are any buildings to process
-	totalBuildings := len(lod2BldIDs) + len(lod3BldIDs)
-	if totalBuildings == 0 {
+	if len(lod2BldIDs)+len(lod3BldIDs) == 0 {
 		utils.Warn.Println("No buildings found in either LOD2 or LOD3 schemas. Nothing to extract.")
 		return nil
 	}
@@ -158,29 +164,8 @@ func runFeatureExtraction(config *config.Config, pool *pgxpool.Pool) error {
 	if pipQueue.Len() > 0 {
 		utils.PrintPipelineQueueInfo(pipQueue.Len(), len(pipQueue.Peek().Jobs), config.Batch)
 	} else {
-		utils.Warn.Printf("Pipeline queue is empty - this shouldn't happen if buildings were found. Check batch creation logic.")
-		// Continue anyway - workers will just have no work to do
-	}
-
-	// Create pipeline channel
-	pipChan := make(chan *process.Pipeline, pipQueue.Len())
-	for !pipQueue.IsEmpty() {
-		pipeline := pipQueue.Dequeue()
-		if pipeline != nil {
-			pipChan <- pipeline
-		}
-	}
-	close(pipChan)
-
-	// Start workers
-	numWorkers := config.Batch.Threads
-	var wg sync.WaitGroup
-	for i := 1; i <= numWorkers; i++ {
-		wg.Add(1)
-		worker := process.NewWorker(i)
-		go worker.Start(pipChan, pool, &wg, config)
+		utils.Warn.Printf("Pipeline queue is empty - this shouldn't happen if buildings were found.")
 	}
-	wg.Wait()
 
-	return nil
+	return process.RunPipelineQueue(pipQueue, pool, config)
 }

environment/docker-compose.yml

@@ -13,5 +13,5 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"  # Linux compatibility
     ports:
-      - "8080:8080"        # map port 8080 for application access
+      - "${HOST_PORT}:5000"        # map host port (configurable) to container port 8080
     command: bash         # start an interactive shell for development

environment/docker.env

@@ -28,6 +28,10 @@ COUNTRY=germany
 # Note: These countries are available based on the TABULA and EPISCOPE project data. Therefore running pipeline on other countries is not supported.
 
 
+# Application port mapping (host port -> container 8080)
+# Change this if 8080 collides with another service
+HOST_PORT=5000
+
 # Database connection settings (Connect to host PostgreSQL)
 DB_HOST=host.docker.internal
 DB_PORT=5432

internal/db/setup.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"sync"
 
 	"github.com/THD-Spatial/City2TABULA/internal/config"
 	"github.com/THD-Spatial/City2TABULA/internal/importer"
@@ -23,7 +22,6 @@ type SetupOperation struct {
 
 // CreateCompleteDatabase creates the complete City2TABULA database with CityDB infrastructure
 func CreateCompleteDatabase(config *config.Config, conn *pgxpool.Pool) error {
-	utils.Info.Println("Creating complete City2TABULA database...")
 
 	// Step 1: Create CityDB infrastructure
 	if err := CreateCityDB(config); err != nil {
@@ -40,7 +38,6 @@ func CreateCompleteDatabase(config *config.Config, conn *pgxpool.Pool) error {
 		return fmt.Errorf("failed to import data: %w", err)
 	}
 
-	utils.Debug.Println("Complete database created successfully")
 	return nil
 }
 
@@ -99,23 +96,9 @@ func RunCity2TabulaDBSetup(config *config.Config, conn *pgxpool.Pool) error {
 		return fmt.Errorf("failed to setup main DB queue: %w", err)
 	}
 
-	mainPipelineChan := make(chan *process.Pipeline, mainPipelineQueue.Len())
-	for !mainPipelineQueue.IsEmpty() {
-		pipeline := mainPipelineQueue.Dequeue()
-		if pipeline != nil {
-			mainPipelineChan <- pipeline
-		}
-	}
-	close(mainPipelineChan)
-
-	numWorkers := config.Batch.Threads
-	var wg sync.WaitGroup
-	for i := 1; i <= numWorkers; i++ {
-		wg.Add(1)
-		worker := process.NewWorker(i)
-		go worker.Start(mainPipelineChan, conn, &wg, config)
+	if err := process.RunPipelineQueue(mainPipelineQueue, conn, config); err != nil {
+		return fmt.Errorf("main DB setup failed: %w", err)
 	}
-	wg.Wait()
 
 	utils.Info.Println("Main database setup completed")
 
@@ -125,21 +108,9 @@ func RunCity2TabulaDBSetup(config *config.Config, conn *pgxpool.Pool) error {
 		return fmt.Errorf("failed to setup supplementary DB queue: %w", err)
 	}
 
-	supplementaryPipelineChan := make(chan *process.Pipeline, supplementaryPipelineQueue.Len())
-	for !supplementaryPipelineQueue.IsEmpty() {
-		pipeline := supplementaryPipelineQueue.Dequeue()
-		if pipeline != nil {
-			supplementaryPipelineChan <- pipeline
-		}
-	}
-	close(supplementaryPipelineChan)
-
-	for i := 1; i <= numWorkers; i++ {
-		wg.Add(1)
-		worker := process.NewWorker(i)
-		go worker.Start(supplementaryPipelineChan, conn, &wg, config)
+	if err := process.RunPipelineQueue(supplementaryPipelineQueue, conn, config); err != nil {
+		return fmt.Errorf("supplementary DB setup failed: %w", err)
 	}
-	wg.Wait()
 
 	utils.Info.Println("Supplementary database setup completed")
 

internal/process/worker.go

@@ -28,3 +28,26 @@ func (w *Worker) Start(pipelineChan <-chan *Pipeline, conn *pgxpool.Pool, wg *sy
 		}
 	}
 }
+
+// RunPipelineQueue drains a PipelineQueue into a channel and runs it with workers.
+func RunPipelineQueue(queue *PipelineQueue, conn *pgxpool.Pool, cfg *config.Config) error {
+	if queue.IsEmpty() {
+		return nil
+	}
+
+	pipChan := make(chan *Pipeline, queue.Len())
+	for !queue.IsEmpty() {
+		if p := queue.Dequeue(); p != nil {
+			pipChan <- p
+		}
+	}
+	close(pipChan)
+
+	var wg sync.WaitGroup
+	for i := 1; i <= cfg.Batch.Threads; i++ {
+		wg.Add(1)
+		go NewWorker(i).Start(pipChan, conn, &wg, cfg)
+	}
+	wg.Wait()
+	return nil
+}

internal/utils/citydb.go

@@ -108,5 +108,11 @@ func ExecuteCityDBScript(cfg *config.Config, sqlFilePath, schemaName string) err
 	if len(out) > 0 {
 		Debug.Printf("psql output:\n%s", string(out))
 	}
-	return err
+	if err != nil {
+		if strings.Contains(string(out), "already exists") {
+			return fmt.Errorf("schema already exists (psql output: %s): %w", strings.TrimSpace(string(out)), err)
+		}
+		return fmt.Errorf("failed to execute CityDB script %s: %w", sqlFilePath, err)
+	}
+	return nil
 }

makefile

@@ -7,45 +7,49 @@ help: ## Show this help message
 	@echo "City2TABULA Make Commands"
 	@echo "========================"
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+	@echo ""
+	@echo "Recommended Workflow:"
+	@echo "  1. make configure        - Set country, DB credentials in environment/docker.env"
+	@echo "  2. make build            - Build the Docker image"
+	@echo "  3. make create-db        - Create database schemas and import CityDB data"
+	@echo "     (If database already exists, run: make reset-db)"
+	@echo "  4. make extract-features - Run the feature extraction pipeline"
+	@echo ""
 
 ##@ Docker Environment
 build: ## Build the Docker environment
-	cd environment && docker compose build --no-cache
+	cd environment && docker compose --env-file docker.env build --no-cache
 
 up: ## Start the Docker environment
-	cd environment && docker compose up -d
+	cd environment && docker compose --env-file docker.env up -d
 
 down: ## Stop the Docker environment
-	cd environment && docker compose down
+	cd environment && docker compose --env-file docker.env down
 
 logs: ## View Docker logs
-	cd environment && docker compose logs -f
+	cd environment && docker compose --env-file docker.env logs -f
 
 status: ## Check container status
-	cd environment && docker compose ps
+	cd environment && docker compose --env-file docker.env ps
 
 ##@ Application Commands
 dev: ## Start development environment with shell
-	cd environment && docker compose up -d && docker exec -it city2tabula-environment bash
+	cd environment && docker compose --env-file docker.env up -d && docker exec -it city2tabula-environment bash
 
 create-db: up ## Create database and setup schemas
-	cd environment && docker exec -it city2tabula-environment ./city2tabula -create_db
+	cd environment && docker exec -it city2tabula-environment ./city2tabula -create-db
 
 extract-features: up ## Extract building features
-	cd environment && docker exec -it city2tabula-environment ./city2tabula -extract_features
+	cd environment && docker exec -it city2tabula-environment ./city2tabula -extract-features
 
 reset-db: up ## Reset the entire database
-	cd environment && docker exec -it city2tabula-environment ./city2tabula -reset_all
+	cd environment && docker exec -it city2tabula-environment ./city2tabula -reset-all
 
 ##@ Complete Workflows
 configure: ## Interactive configuration: select country and enter password
 	@echo "City2TABULA Interactive Configuration"
 	@echo "========================================"
 	@echo ""
-	@echo "Copying base environment configuration..."
-	@cp environment/docker.env .env
-	@echo "Base configuration copied!"
-	@echo ""
 	@echo "Available Countries:"
 	@echo "======================"
 	@echo " 1) austria       - SRID: 31256 (MGI / Austria GK East)"
@@ -68,7 +72,7 @@ configure: ## Interactive configuration: select country and enter password
 	@echo "18) sweden        - SRID: 3006  (SWEREF99 TM)"
 	@echo "19) united_kingdom - SRID: 27700 (OSGB 1936 / British National Grid)"
 	@echo ""
-	@read -p "Select country (1-19): " choice; \
+	@read -p "For which country would you like to configure City2TABULA? Enter a number (1-19): " choice; \
 	case $$choice in \
 		1) COUNTRY="austria"; SRID="31256"; SRS_NAME="MGI / Austria GK East" ;; \
 		2) COUNTRY="belgium"; SRID="31370"; SRS_NAME="Belgian Lambert 72" ;; \
@@ -96,28 +100,27 @@ configure: ## Interactive configuration: select country and enter password
 	echo ""; \
 	echo "Database Configuration:"; \
 	echo "========================="; \
-	echo -n "Enter PostgreSQL username [default: postgres]: "; \
-	read pg_user; \
+	read -p "Enter PostgreSQL username [default: postgres]: " pg_user; \
 	if [ -z "$$pg_user" ]; then pg_user="postgres"; fi; \
 	echo -n "Enter PostgreSQL password: "; \
 	stty -echo; \
 	read pg_password; \
 	stty echo; \
 	echo ""; \
 	echo ""; \
-	echo "Updating configuration file..."; \
+	echo "Updating environment/docker.env..."; \
 	if [ "$$(uname)" = "Darwin" ]; then \
-		sed -i '' "s/^COUNTRY=.*/COUNTRY=$$COUNTRY/" .env; \
-		sed -i '' "s/^CITYDB_SRID=.*/CITYDB_SRID=$$SRID/" .env; \
-		sed -i '' "s|^CITYDB_SRS_NAME=.*|CITYDB_SRS_NAME=$$SRS_NAME|" .env; \
-		sed -i '' "s/^DB_USER=.*/DB_USER=$$pg_user/" .env; \
-		sed -i '' "s/<your_pg_password>/$$pg_password/" .env; \
+		sed -i '' "s/^COUNTRY=.*/COUNTRY=$$COUNTRY/" environment/docker.env; \
+		sed -i '' "s/^CITYDB_SRID=.*/CITYDB_SRID=$$SRID/" environment/docker.env; \
+		sed -i '' "s|^CITYDB_SRS_NAME=.*|CITYDB_SRS_NAME=$$SRS_NAME|" environment/docker.env; \
+		sed -i '' "s/^DB_USER=.*/DB_USER=$$pg_user/" environment/docker.env; \
+		sed -i '' "s/^DB_PASSWORD=.*/DB_PASSWORD=$$pg_password/" environment/docker.env; \
 	else \
-		sed -i "s/^COUNTRY=.*/COUNTRY=$$COUNTRY/" .env; \
-		sed -i "s/^CITYDB_SRID=.*/CITYDB_SRID=$$SRID/" .env; \
-		sed -i "s|^CITYDB_SRS_NAME=.*|CITYDB_SRS_NAME=$$SRS_NAME|" .env; \
-		sed -i "s/^DB_USER=.*/DB_USER=$$pg_user/" .env; \
-		sed -i "s/<your_pg_password>/$$pg_password/" .env; \
+		sed -i "s/^COUNTRY=.*/COUNTRY=$$COUNTRY/" environment/docker.env; \
+		sed -i "s/^CITYDB_SRID=.*/CITYDB_SRID=$$SRID/" environment/docker.env; \
+		sed -i "s|^CITYDB_SRS_NAME=.*|CITYDB_SRS_NAME=$$SRS_NAME|" environment/docker.env; \
+		sed -i "s/^DB_USER=.*/DB_USER=$$pg_user/" environment/docker.env; \
+		sed -i "s/^DB_PASSWORD=.*/DB_PASSWORD=$$pg_password/" environment/docker.env; \
 	fi; \
 	echo "Configuration completed!"; \
 	echo ""; \
@@ -129,22 +132,21 @@ configure: ## Interactive configuration: select country and enter password
 	echo "Database: Configured"; \
 	echo ""; \
 	echo "Next steps:"; \
-	echo "- Place your data in data/lod2/$$COUNTRY/ and data/lod3/$$COUNTRY/"
+	echo "- Place your data in data/lod2/$$COUNTRY/ and data/lod3/$$COUNTRY/"; \
 	echo "- Run 'make up' to start containers"; \
-	echo "- Run 'make dev' to access development shell"; \
+	echo "- Run 'make dev' to access development shell"
 
 
-setup: build configure ## Build environment, copy .env, and start containers
+setup: build configure ## Build environment, configure, and start containers
 	@$(MAKE) up
 	@echo "Environment is ready! Run 'make dev' to access the shell"
-	@echo "Don't forget to edit .env with your PostgreSQL password if you haven't already"
 
 quick-start: setup create-db extract-features ## Complete setup and processing
 	@echo "Quick start complete!"
 
 ##@ Cleanup
 clean: ## Stop containers and remove volumes
-	cd environment && docker compose down -v
+	cd environment && docker compose --env-file docker.env down -v
 
 clean-all: ## Remove containers, volumes, and images
-	cd environment && docker compose down -v --rmi all
+	cd environment && docker compose --env-file docker.env down -v --rmi all