Merge pull request #1453 from Altinity/feature_tables_parts_and_partitions

add --list-parts and --list-partitions to tables command,

Author: Slach

Manual.md

@@ -4,16 +4,22 @@ NAME:
    clickhouse-backup tables - List of tables, exclude skip_tables
 
 USAGE:
-   clickhouse-backup tables [--tables=<db>.<table>] [--remote-backup=<backup-name>] [--local-backup=<backup-name>] [-f, --format=<text|json|yaml|csv|tsv>] [--all]
+   clickhouse-backup tables [--tables=<db>.<table>] [--remote-backup=<backup-name>] [--local-backup=<backup-name>] [-f, --format=<text|json|yaml|csv|tsv>] [--all] [--parts] [--partitions]
 
 OPTIONS:
-   --config value, -c value                   Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
-   --environment-override value, --env value  override any environment variable via CLI parameter
-   --all, -a                                  Print table even when match with skip_tables pattern
-   --table value, --tables value, -t value    List tables only match with table name patterns, separated by comma, allow ? and * as wildcard
-   --remote-backup value                      List tables from a remote backup, including per-table size and parts count
-   --local-backup value                       List tables from a local backup (read from disk, no live ClickHouse query), including per-table size and parts count
-   --format value, -f value                   Output format (text|json|yaml|csv|tsv)
+   --config value, -c value                         Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
+   --environment-override value, --env value        override any environment variable via CLI parameter
+   --all, -a                                        Print table even when match with skip_tables pattern
+   --table value, --tables value, -t value          List tables only match with table name patterns, separated by comma, allow ? and * as wildcard
+   --remote-backup value                            List tables from a remote backup, including per-table size and parts count
+   --local-backup value                             List tables from a local backup (read from disk, no live ClickHouse query), including per-table size and parts count
+   --format value, -f value                         Output format (text|json|yaml|csv|tsv)
+   --parts system.parts, --list-parts system.parts  Also list every physical part for each table (name, partition_id, size)
+Against the live server, reads name/partition_id/bytes_on_disk from system.parts
+Against --local-backup/--remote-backup, reads part names from backup metadata (partition_id derived from the name, no size available)
+   --partitions system.parts, --list-partitions system.parts  Also list the distinct partitions for each table (partition_id, partition, parts count, size), aggregated from parts
+Against the live server, reads partition_id/partition/parts/size from system.parts
+Against --local-backup/--remote-backup, derives partition_id and parts count from part names (no partition value or per-partition size available)
    
 ```
 ### CLI command - create
@@ -208,6 +214,7 @@ Look at the system.parts partition and partition_id fields for details https://c
    --restore-schema-as-attach                                                        Use DETACH/ATTACH instead of DROP/CREATE for schema restoration
    --replicated-copy-to-detached                                                     Copy data to detached folder for Replicated*MergeTree tables but skip ATTACH PART step
    --skip-empty-tables                                                               Skip restoring tables that have no data (empty tables with only schema)
+   --rebind-replica-path-if-exists                                                   Override clickhouse.rebind_replica_path_if_exists, rebind a restored ReplicatedMergeTree to default_replica_path when the original ZK path still has leftover state but our replica entry is absent
    
 ```
 ### CLI command - restore_remote
@@ -246,6 +253,7 @@ Look at the system.parts partition and partition_id fields for details https://c
    --restore-schema-as-attach                                                        Use DETACH/ATTACH instead of DROP/CREATE for schema restoration
    --hardlink-exists-files                                                           Create hardlinks for existing files instead of downloading
    --skip-empty-tables                                                               Skip restoring tables that have no data (empty tables with only schema)
+   --rebind-replica-path-if-exists                                                   Override clickhouse.rebind_replica_path_if_exists, rebind a restored ReplicatedMergeTree to default_replica_path when the original ZK path still has leftover state but our replica entry is absent
    
 ```
 ### CLI command - delete

ReadMe.md

@@ -486,6 +486,8 @@ Print list of tables: `curl -s localhost:7171/backup/tables | jq .`, exclude pat
 - Optional query argument `table` works the same as the `--table=pattern` CLI argument.
 - Optional query argument `remote_backup` (or `remote-backup`) works the same as the `--remote-backup=name` CLI argument. The response then includes per-table `size`, `total_bytes`, `parts`, and `disks` (JSON array of disk names) fields read from the remote backup metadata.
 - Optional query argument `local_backup` (or `local-backup`) works the same as the `--local-backup=name` CLI argument: it lists tables from a local backup directly from disk (no live ClickHouse query required), with `size`, `total_bytes`, `parts`, and `disks` (JSON array) fields.
+- Optional boolean query argument `list_parts` (alias `parts`) works the same as the `--list-parts`/`--parts` CLI argument: it attaches a `parts_list` array per table (`name`, `partition_id`, and against the live server also `total_bytes`/`size`). Works standalone against the live server or together with `remote_backup`/`local_backup`; against a remote/local backup, `partition_id` is derived from the `_`-delimited prefix of each part's name (no size available there).
+- Optional boolean query argument `partitions` (alias `list_partitions`) works the same as the `--partitions`/`--list-partitions` CLI argument: it attaches a `partitions` array per table (`partition_id`, and against the live server also `partition`/`parts`/`total_bytes`/`size`), aggregated from parts. Independent of `list_parts`; against a remote/local backup, `partition_id` is derived the same way as `list_parts` (no partition value or size available there).
 
 ### GET /backup/tables/all
 
@@ -494,6 +496,7 @@ Print list of tables: `curl -s localhost:7171/backup/tables/all | jq .`, ignore
 - Optional query argument `table` works the same as the `--table=pattern` CLI argument.
 - Optional query argument `remote_backup` (or `remote-backup`) works the same as the `--remote-backup=name` CLI argument; response shape matches `GET /backup/tables` with the remote-backup parameter.
 - Optional query argument `local_backup` (or `local-backup`) works the same as the `--local-backup=name` CLI argument; response shape matches `GET /backup/tables` with the local-backup parameter.
+- Optional boolean query arguments `list_parts`/`parts` and `partitions`/`list_partitions` work the same as on `GET /backup/tables`.
 
 ### POST /backup/create
 
@@ -732,16 +735,22 @@ NAME:
    clickhouse-backup tables - List of tables, exclude skip_tables
 
 USAGE:
-   clickhouse-backup tables [--tables=<db>.<table>] [--remote-backup=<backup-name>] [--local-backup=<backup-name>] [-f, --format=<text|json|yaml|csv|tsv>] [--all]
+   clickhouse-backup tables [--tables=<db>.<table>] [--remote-backup=<backup-name>] [--local-backup=<backup-name>] [-f, --format=<text|json|yaml|csv|tsv>] [--all] [--parts] [--partitions]
 
 OPTIONS:
-   --config value, -c value                   Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
-   --environment-override value, --env value  override any environment variable via CLI parameter
-   --all, -a                                  Print table even when match with skip_tables pattern
-   --table value, --tables value, -t value    List tables only match with table name patterns, separated by comma, allow ? and * as wildcard
-   --remote-backup value                      List tables from a remote backup, including per-table size and parts count
-   --local-backup value                       List tables from a local backup (read from disk, no live ClickHouse query), including per-table size and parts count
-   --format value, -f value                   Output format (text|json|yaml|csv|tsv)
+   --config value, -c value                         Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
+   --environment-override value, --env value        override any environment variable via CLI parameter
+   --all, -a                                        Print table even when match with skip_tables pattern
+   --table value, --tables value, -t value          List tables only match with table name patterns, separated by comma, allow ? and * as wildcard
+   --remote-backup value                            List tables from a remote backup, including per-table size and parts count
+   --local-backup value                             List tables from a local backup (read from disk, no live ClickHouse query), including per-table size and parts count
+   --format value, -f value                         Output format (text|json|yaml|csv|tsv)
+   --parts system.parts, --list-parts system.parts  Also list every physical part for each table (name, partition_id, size)
+Against the live server, reads name/partition_id/bytes_on_disk from system.parts
+Against --local-backup/--remote-backup, reads part names from backup metadata (partition_id derived from the name, no size available)
+   --partitions system.parts, --list-partitions system.parts  Also list the distinct partitions for each table (partition_id, partition, parts count, size), aggregated from parts
+Against the live server, reads partition_id/partition/parts/size from system.parts
+Against --local-backup/--remote-backup, derives partition_id and parts count from part names (no partition value or per-partition size available)
    
 ```
 ### CLI command - create

cmd/clickhouse-backup/main.go

@@ -88,10 +88,10 @@ func main() {
 		{
 			Name:      "tables",
 			Usage:     "List of tables, exclude skip_tables",
-			UsageText: "clickhouse-backup tables [--tables=<db>.<table>] [--remote-backup=<backup-name>] [--local-backup=<backup-name>] [-f, --format=<text|json|yaml|csv|tsv>] [--all]",
+			UsageText: "clickhouse-backup tables [--tables=<db>.<table>] [--remote-backup=<backup-name>] [--local-backup=<backup-name>] [-f, --format=<text|json|yaml|csv|tsv>] [--all] [--parts] [--partitions]",
 			Action: func(c *cli.Context) error {
 				b := backup.NewBackuper(config.GetConfigFromCli(c))
-				return b.PrintTables(c.Bool("all"), c.String("table"), c.String("remote-backup"), c.String("local-backup"), c.String("format"))
+				return b.PrintTables(c.Bool("all"), c.String("table"), c.String("remote-backup"), c.String("local-backup"), c.String("format"), c.Bool("parts"), c.Bool("partitions"))
 			},
 			Flags: append(cliapp.Flags,
 				cli.BoolFlag{
@@ -119,6 +119,20 @@ func main() {
 					Hidden: false,
 					Usage:  "Output format (text|json|yaml|csv|tsv)",
 				},
+				cli.BoolFlag{
+					Name:   "parts, list-parts",
+					Hidden: false,
+					Usage: "Also list every physical part for each table (name, partition_id, size)\n" +
+						"Against the live server, reads name/partition_id/bytes_on_disk from `system.parts`\n" +
+						"Against --local-backup/--remote-backup, reads part names from backup metadata (partition_id derived from the name, no size available)",
+				},
+				cli.BoolFlag{
+					Name:   "partitions, list-partitions",
+					Hidden: false,
+					Usage: "Also list the distinct partitions for each table (partition_id, partition, parts count, size), aggregated from parts\n" +
+						"Against the live server, reads partition_id/partition/parts/size from `system.parts`\n" +
+						"Against --local-backup/--remote-backup, derives partition_id and parts count from part names (no partition value or per-partition size available)",
+				},
 			),
 		},
 		{

generate_manual.sh

@@ -4,16 +4,32 @@ CLICKHOUSE_BACKUP_BIN=${CLICKHOUSE_BACKUP_BIN:-build/$(go env GOOS)/$(go env GOA
 make clean "${CLICKHOUSE_BACKUP_BIN}" >&2
 
 # Extract command list dynamically from `--help`. Skip the built-in `help, h` entry.
+# New commands (e.g. `rebase`) need no changes here - they're picked up automatically.
 mapfile -t cmds < <(
   go run ./cmd/clickhouse-backup/ --help 2>/dev/null \
     | awk '/^COMMANDS:/{flag=1; next} /^GLOBAL OPTIONS:/{flag=0} flag && NF{print $1}' \
     | sed 's/,$//' \
     | grep -vE '^(help|h)$'
 )
 
+manual_file=$(mktemp)
+trap 'rm -f "${manual_file}"' EXIT
+
 for cmd in "${cmds[@]}"; do
   echo "### CLI command - ${cmd}"
   echo '```'
   "${CLICKHOUSE_BACKUP_BIN}" help "${cmd}"
   echo '```'
-done
+done > "${manual_file}"
+
+cp "${manual_file}" Manual.md
+
+# README.md embeds the same content under "## Common CLI Usage" - keep it in sync so the two
+# docs can't drift apart again.
+readme_tmp=$(mktemp)
+trap 'rm -f "${manual_file}" "${readme_tmp}"' EXIT
+awk '/^## Common CLI Usage$/{print; print ""; exit} {print}' README.md > "${readme_tmp}"
+cat "${manual_file}" >> "${readme_tmp}"
+mv "${readme_tmp}" README.md
+
+echo "Updated Manual.md and README.md's 'Common CLI Usage' section." >&2

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/klauspost/compress v1.19.0
 	github.com/klauspost/pgzip v1.2.6
-	github.com/mattn/go-shellwords v1.0.13
+	github.com/mattn/go-shellwords v1.0.14
 	github.com/mholt/archives v0.1.5
 	github.com/otiai10/copy v1.14.1
 	github.com/pkg/errors v0.9.1
@@ -51,7 +51,7 @@ require (
 	golang.org/x/mod v0.38.0
 	golang.org/x/sync v0.22.0
 	golang.org/x/text v0.40.0
-	google.golang.org/api v0.287.1
+	google.golang.org/api v0.288.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -66,9 +66,9 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
 	github.com/ClickHouse/ch-go v0.73.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.58.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/STARRY-S/zip v0.2.3 // indirect
 	github.com/andybalholm/brotli v1.2.2 // indirect
@@ -137,7 +137,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.69.0 // indirect
+	github.com/prometheus/common v0.70.0 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/segmentio/asm v1.2.1 // indirect