Skip to content

Commit c212b99

Browse files
committed
Merge branch 'feat/sync-stderr-logging' into 'master'
feat(engine): document sync stderr logging, add log fallback Closes #711 See merge request postgres-ai/database-lab!1146
2 parents f0d5402 + 87221fd commit c212b99

4 files changed

Lines changed: 111 additions & 4 deletions

File tree

engine/configs/config.example.physical_generic.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ retrieval: # Data retrieval: initial sync and ongoing updates. Two methods:
7171
maxRetries: 200 # Max retries before giving up
7272
configs: # Additional Postgres configuration for sync container
7373
shared_buffers: 2GB # Bigger buffer pool helps avoid lagging behind the source
74+
# Uncomment to ship sync-container Postgres logs to the Docker logging driver
75+
# (e.g. for ingestion by an external log collector) instead of CSV files in PGDATA/log.
76+
# Note: disables the CSV-based diagnostics fallback for the sync container.
77+
# Optionally append "jsonlog" to log_destination for structured JSON output (Postgres 15+).
78+
# log_destination: stderr
79+
# logging_collector: "off" # quoted: go-yaml.v2 (YAML 1.1) treats bare off as boolean false
7480
recovery: # Legacy recovery.conf options; only for Postgres 11 or older
7581
# standby_mode: on
7682
# recovery_target_timeline: 'latest'
@@ -98,6 +104,12 @@ retrieval: # Data retrieval: initial sync and ongoing updates. Two methods:
98104
inline: "" # Direct SQL queries to execute after scripts from 'queryPath'. Supports multiple statements separated by semicolons
99105
configs: # Postgres configuration overrides for promotion container
100106
shared_buffers: 2GB
107+
# Uncomment to ship promotion-container Postgres logs to the Docker logging driver
108+
# (e.g. for ingestion by an external log collector) instead of CSV files in PGDATA/log.
109+
# Note: disables the CSV-based diagnostics fallback for the promotion container.
110+
# Optionally append "jsonlog" to log_destination for structured JSON output (Postgres 15+).
111+
# log_destination: stderr
112+
# logging_collector: "off" # quoted: go-yaml.v2 (YAML 1.1) treats bare off as boolean false
101113
recovery: # Legacy recovery.conf configuration options; only applicable for Postgres 11 or earlier versions
102114
# recovery_target: 'immediate'
103115
# recovery_target_action: 'promote'

engine/configs/config.example.physical_pgbackrest.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ retrieval: # Data retrieval: initial sync and ongoing updates. Two methods:
7171
maxRetries: 200 # Max retries before giving up
7272
configs: # Additional Postgres configuration for sync container
7373
shared_buffers: 2GB # Bigger buffer pool helps avoid lagging behind the source
74+
# Uncomment to ship sync-container Postgres logs to the Docker logging driver
75+
# (e.g. for ingestion by an external log collector) instead of CSV files in PGDATA/log.
76+
# Note: disables the CSV-based diagnostics fallback for the sync container.
77+
# Optionally append "jsonlog" to log_destination for structured JSON output (Postgres 15+).
78+
# log_destination: stderr
79+
# logging_collector: "off" # quoted: go-yaml.v2 (YAML 1.1) treats bare off as boolean false
7480
recovery: # Legacy recovery.conf options; only for Postgres 11 or older
7581
# standby_mode: on
7682
# recovery_target_timeline: 'latest'
@@ -112,6 +118,12 @@ retrieval: # Data retrieval: initial sync and ongoing updates. Two methods:
112118
inline: "" # Direct SQL queries to execute after scripts from 'queryPath'. Supports multiple statements separated by semicolons
113119
configs: # Postgres configuration overrides for promotion container
114120
shared_buffers: 2GB
121+
# Uncomment to ship promotion-container Postgres logs to the Docker logging driver
122+
# (e.g. for ingestion by an external log collector) instead of CSV files in PGDATA/log.
123+
# Note: disables the CSV-based diagnostics fallback for the promotion container.
124+
# Optionally append "jsonlog" to log_destination for structured JSON output (Postgres 15+).
125+
# log_destination: stderr
126+
# logging_collector: "off" # quoted: go-yaml.v2 (YAML 1.1) treats bare off as boolean false
115127
recovery: # Legacy recovery.conf configuration options; only applicable for Postgres 11 or earlier versions
116128
# recovery_target: 'immediate'
117129
# recovery_target_action: 'promote'

engine/configs/config.example.physical_walg.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ retrieval: # Data retrieval: initial sync and ongoing updates. Two methods:
7171
maxRetries: 200 # Max retries before giving up
7272
configs: # Additional Postgres configuration for sync container
7373
shared_buffers: 2GB # Bigger buffer pool helps avoid lagging behind the source
74+
# Uncomment to ship sync-container Postgres logs to the Docker logging driver
75+
# (e.g. for ingestion by an external log collector) instead of CSV files in PGDATA/log.
76+
# Note: disables the CSV-based diagnostics fallback for the sync container.
77+
# Optionally append "jsonlog" to log_destination for structured JSON output (Postgres 15+).
78+
# log_destination: stderr
79+
# logging_collector: "off" # quoted: go-yaml.v2 (YAML 1.1) treats bare off as boolean false
7480
recovery: # Legacy recovery.conf options; only for Postgres 11 or older
7581
# standby_mode: on
7682
# recovery_target_timeline: 'latest'
@@ -98,6 +104,12 @@ retrieval: # Data retrieval: initial sync and ongoing updates. Two methods:
98104
inline: "" # Direct SQL queries to execute after scripts from 'queryPath'. Supports multiple statements separated by semicolons
99105
configs: # Postgres configuration overrides for promotion container
100106
shared_buffers: 2GB
107+
# Uncomment to ship promotion-container Postgres logs to the Docker logging driver
108+
# (e.g. for ingestion by an external log collector) instead of CSV files in PGDATA/log.
109+
# Note: disables the CSV-based diagnostics fallback for the promotion container.
110+
# Optionally append "jsonlog" to log_destination for structured JSON output (Postgres 15+).
111+
# log_destination: stderr
112+
# logging_collector: "off" # quoted: go-yaml.v2 (YAML 1.1) treats bare off as boolean false
101113
recovery: # Legacy recovery.conf configuration options; only applicable for Postgres 11 or earlier versions
102114
# recovery_target: 'immediate'
103115
# recovery_target_action: 'promote'

engine/internal/retrieval/engine/postgres/tools/tools.go

Lines changed: 75 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ const (
6262

6363
// defaultLogsDir defines default location of diagnostic logs on the host machine.
6464
defaultLogsDir = "~/.dblab/engine/logs"
65+
66+
// defaultPostgresLogTailLines is the number of trailing log lines fetched when
67+
// reporting Postgres logs after a failed health check.
68+
defaultPostgresLogTailLines = 20
6569
)
6670

6771
// ErrHealthCheck defines a health check errors.
@@ -473,18 +477,85 @@ func printPostgresLogsHint(ctx context.Context, dockerClient *client.Client, con
473477
"check (%s) on DLE machine.\n", logsHostDir))
474478
}
475479

476-
// PrintLastPostgresLogs prints Postgres container logs.
480+
// PrintLastPostgresLogs prints the last lines of Postgres container logs.
481+
// It reads the newest CSV produced by the logging collector under
482+
// clonePath/log; if no CSV is present or the CSV read fails, it falls
483+
// back to the container stdout/stderr stream.
477484
func PrintLastPostgresLogs(ctx context.Context, dockerClient *client.Client, containerID, clonePath string) {
478-
command := []string{"bash", "-c", "tail -n 20 $(ls -t " + clonePath + "/log/*.csv | tail -n 1)"}
485+
if output := tryTailCSV(ctx, dockerClient, containerID, clonePath); output != "" {
486+
log.Msg("Postgres logs: ", output)
487+
return
488+
}
479489

480-
output, err := ExecCommandWithOutput(ctx, dockerClient, containerID, container.ExecOptions{Cmd: command})
490+
output, err := tailContainerLogs(ctx, dockerClient, containerID)
481491
if err != nil {
482-
log.Err(errors.Wrap(err, "failed to read Postgres logs"))
492+
if output != "" {
493+
log.Msg("Postgres logs (partial): ", output)
494+
}
495+
496+
log.Err(errors.Wrap(err, "failed to read Postgres container logs"))
497+
498+
return
499+
}
500+
501+
if output == "" {
502+
log.Msg("Postgres logs: (no output)")
503+
return
483504
}
484505

485506
log.Msg("Postgres logs: ", output)
486507
}
487508

509+
// tryTailCSV returns the tail of the newest CSV log file under clonePath/log,
510+
// or "" if no CSV is found or the read fails. The list and tail run in a
511+
// single exec to reduce (but not eliminate) the TOCTOU window with the
512+
// Postgres logging collector rotating the file between calls.
513+
func tryTailCSV(ctx context.Context, dockerClient *client.Client, containerID, clonePath string) string {
514+
script := `f=$(ls -t "$1"/log/*.csv 2>/dev/null | head -n 1); ` +
515+
`[ -z "$f" ] && exit 0; tail -n "$2" -- "$f"`
516+
cmd := []string{"bash", "-c", script, "_", clonePath, strconv.Itoa(defaultPostgresLogTailLines)}
517+
518+
output, err := ExecCommandWithOutput(ctx, dockerClient, containerID, container.ExecOptions{Cmd: cmd})
519+
if err != nil {
520+
log.Warn("failed to read Postgres CSV logs: ", err)
521+
return ""
522+
}
523+
524+
if strings.TrimSpace(output) == "" {
525+
return ""
526+
}
527+
528+
return output
529+
}
530+
531+
// tailContainerLogs fetches the last defaultPostgresLogTailLines entries from
532+
// the container's stdout/stderr stream. When the read fails partway, the
533+
// partial buffer is returned along with the error so the caller can still
534+
// surface diagnostics.
535+
func tailContainerLogs(ctx context.Context, dockerClient *client.Client, containerID string) (string, error) {
536+
reader, err := dockerClient.ContainerLogs(ctx, containerID, container.LogsOptions{
537+
ShowStdout: true,
538+
ShowStderr: true,
539+
Tail: strconv.Itoa(defaultPostgresLogTailLines),
540+
})
541+
if err != nil {
542+
return "", err
543+
}
544+
545+
defer func() {
546+
if err := reader.Close(); err != nil {
547+
log.Err(errors.Wrap(err, "failed to close container logs reader"))
548+
}
549+
}()
550+
551+
buf := &bytes.Buffer{}
552+
if _, err := io.Copy(buf, dlog.NewReader(reader)); err != nil {
553+
return buf.String(), err
554+
}
555+
556+
return buf.String(), nil
557+
}
558+
488559
// StopContainer stops container.
489560
func StopContainer(ctx context.Context, dockerClient *client.Client, containerID string, stopTimeout int) {
490561
log.Msg(fmt.Sprintf("Stopping container ID: %v", containerID))

0 commit comments

Comments
 (0)