Skip to content

Commit c1dcab3

Browse files
rostalanrostalan
committed
more logging
1 parent a298bdb commit c1dcab3

1 file changed

Lines changed: 250 additions & 71 deletions

File tree

‎workspaces/orchestrator/e2e-tests/tests/specs/test-helpers.ts‎

Lines changed: 250 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -623,9 +623,18 @@ export function runOc(args: string[], timeoutMs = 30_000): string {
623623
return execFileSync("oc", args, {
624624
encoding: "utf-8",
625625
timeout: timeoutMs,
626+
maxBuffer: 32 * 1024 * 1024,
626627
}).trim();
627628
}
628629

630+
function formatOcFailure(err: unknown): string {
631+
if (err instanceof Error) {
632+
const m = err.message.trim();
633+
return m.includes("\n") ? (m.split("\n")[0] ?? m) : m;
634+
}
635+
return String(err);
636+
}
637+
629638
/**
630639
* Best-effort diagnostics bundle for CI when `rhdh.deploy()` fails waiting for RHDH.
631640
* Keep this intentionally stderr-heavy: it should only run on failure paths.
@@ -634,24 +643,30 @@ export function logOrchestratorDeployFailureDiagnostics(
634643
namespace: string,
635644
): void {
636645
const banner = (title: string) => {
637-
// eslint-disable-next-line no-console -- diagnostics for CI triage
638646
console.error(`\n===== [orchestrator-e2e diagnostics] ${title} =====\n`);
639647
};
640648

641649
const safeOc = (args: string[], timeoutMs = 120_000): string | undefined => {
642650
try {
643651
return runOc(args, timeoutMs);
644652
} catch (err) {
645-
// eslint-disable-next-line no-console -- diagnostics for CI triage
646653
console.error(
647-
`[orchestrator-e2e diagnostics] oc ${args.join(" ")} failed:`,
654+
`[orchestrator-e2e diagnostics] oc ${args.join(" ")} failed: ${formatOcFailure(err)}`,
648655
);
649-
// eslint-disable-next-line no-console -- diagnostics for CI triage
650-
console.error(err);
651656
return undefined;
652657
}
653658
};
654659

660+
/** Print `oc` stdout; empty string usually means “no matches” (message on stderr). */
661+
const dumpOc = (out: string | undefined, emptyHint: string) => {
662+
if (out === undefined) return;
663+
if (out.trim().length > 0) {
664+
console.error(out);
665+
} else {
666+
console.error(emptyHint);
667+
}
668+
};
669+
655670
banner(`namespace=${namespace}`);
656671

657672
const hubPod = safeOc([
@@ -667,94 +682,258 @@ export function logOrchestratorDeployFailureDiagnostics(
667682

668683
if (hubPod) {
669684
banner(`redhat-developer-hub pod describe (${hubPod})`);
670-
safeOc(["describe", "pod", "-n", namespace, hubPod], 120_000);
685+
dumpOc(
686+
safeOc(["describe", "pod", "-n", namespace, hubPod], 120_000),
687+
"(describe produced no stdout)",
688+
);
671689

672690
banner(`redhat-developer-hub pod logs (${hubPod}) --all-containers`);
673-
safeOc(
674-
["logs", "-n", namespace, hubPod, "--all-containers", "--tail=400"],
675-
180_000,
691+
dumpOc(
692+
safeOc(
693+
["logs", "-n", namespace, hubPod, "--all-containers", "--tail=400"],
694+
180_000,
695+
),
696+
"(no current container logs on stdout)",
676697
);
677698

678699
banner(
679700
`redhat-developer-hub previous pod logs (${hubPod}) --all-containers`,
680701
);
702+
try {
703+
console.error(
704+
runOc(
705+
[
706+
"logs",
707+
"-n",
708+
namespace,
709+
hubPod,
710+
"--all-containers",
711+
"--previous",
712+
"--tail=400",
713+
],
714+
180_000,
715+
),
716+
);
717+
} catch (err) {
718+
const msg = formatOcFailure(err);
719+
if (/BadRequest|not found|no previous/i.test(msg)) {
720+
console.error(
721+
"(no previous pod/container logs — single revision or init never restarted)",
722+
);
723+
} else {
724+
console.error(`previous logs unavailable: ${msg}`);
725+
}
726+
}
727+
} else {
728+
banner("redhat-developer-hub pod not found via label selector");
729+
dumpOc(
730+
safeOc(["get", "pods", "-n", namespace, "-o", "wide"], 120_000),
731+
"(get pods — empty stdout)",
732+
);
733+
}
734+
735+
banner(
736+
"SonataFlow / workflow / data-index pods (namespace-wide — label selectors vary by OSL version)",
737+
);
738+
dumpOc(
739+
safeOc(["get", "pods", "-n", namespace, "-o", "wide"], 120_000),
740+
"(no pods listed in namespace)",
741+
);
742+
743+
banner(
744+
"sonataflow workflow pods (label selectors — may be empty on some OSL builds)",
745+
);
746+
dumpOc(
747+
safeOc(
748+
[
749+
"get",
750+
"pods",
751+
"-n",
752+
namespace,
753+
"-l",
754+
"sonataflow.org/workflowName=failswitch",
755+
"-o",
756+
"wide",
757+
],
758+
120_000,
759+
),
760+
"(no pods with sonataflow.org/workflowName=failswitch — see namespace-wide list above)",
761+
);
762+
dumpOc(
681763
safeOc(
682764
[
683-
"logs",
765+
"get",
766+
"pods",
684767
"-n",
685768
namespace,
686-
hubPod,
687-
"--all-containers",
688-
"--previous",
689-
"--tail=400",
769+
"-l",
770+
"sonataflow.org/workflowName=greeting",
771+
"-o",
772+
"wide",
690773
],
691-
180_000,
774+
120_000,
775+
),
776+
"(no pods with sonataflow.org/workflowName=greeting — see namespace-wide list above)",
777+
);
778+
779+
banner(
780+
"SonataFlow CRs (oc get sonataflow … -o yaml — persistence vs operator reconcile)",
781+
);
782+
for (const workflow of WORKFLOWS) {
783+
banner(`sonataflow/${workflow} (full YAML)`);
784+
dumpOc(
785+
safeOc(
786+
["get", "sonataflow", workflow, "-n", namespace, "-o", "yaml"],
787+
120_000,
788+
),
789+
`(get sonataflow/${workflow} returned empty stdout)`,
692790
);
693-
} else {
694-
banner("redhat-developer-hub pod not found via label selector");
695-
safeOc(["get", "pods", "-n", namespace, "-o", "wide"], 120_000);
696791
}
697792

698-
banner("sonataflow platform pods (wide)");
699-
safeOc(
700-
[
701-
"get",
702-
"pods",
703-
"-n",
704-
namespace,
705-
"-l",
706-
"app.kubernetes.io/name=logic-operator",
707-
"-o",
708-
"wide",
709-
],
710-
120_000,
793+
banner(
794+
"workflow Deployments (oc describe deployment … — secret/volume mounts on pod template)",
711795
);
712-
safeOc(
713-
[
714-
"get",
715-
"pods",
716-
"-n",
717-
namespace,
718-
"-l",
719-
"app.kubernetes.io/component=sonataflow-platform",
720-
"-o",
721-
"wide",
722-
],
723-
120_000,
796+
for (const workflow of WORKFLOWS) {
797+
banner(`deployment/${workflow} (describe)`);
798+
dumpOc(
799+
safeOc(["describe", "deployment", workflow, "-n", namespace], 120_000),
800+
`(describe deployment/${workflow} returned empty stdout)`,
801+
);
802+
}
803+
804+
banner(
805+
"Services + Endpoints (workflow — empty endpoints => DNS ok but no ready backends)",
806+
);
807+
dumpOc(
808+
safeOc(
809+
["get", "svc", "failswitch", "greeting", "-n", namespace, "-o", "wide"],
810+
60_000,
811+
),
812+
"(get svc failswitch greeting — empty stdout)",
724813
);
814+
dumpOc(
815+
safeOc(
816+
[
817+
"get",
818+
"endpoints",
819+
"failswitch",
820+
"greeting",
821+
"-n",
822+
namespace,
823+
"-o",
824+
"wide",
825+
],
826+
60_000,
827+
),
828+
"(get endpoints — empty stdout)",
829+
);
830+
for (const svc of ["failswitch", "greeting"] as const) {
831+
dumpOc(
832+
safeOc(
833+
[
834+
"get",
835+
"endpointslices.discovery.k8s.io",
836+
"-n",
837+
namespace,
838+
"-l",
839+
`kubernetes.io/service-name=${svc}`,
840+
"-o",
841+
"wide",
842+
],
843+
60_000,
844+
),
845+
`(no EndpointSlices for service ${svc} — empty stdout)`,
846+
);
847+
}
725848

726-
banner("sonataflow workflow pods (failswitch/greeting)");
727-
safeOc(
728-
[
729-
"get",
730-
"pods",
731-
"-n",
732-
namespace,
733-
"-l",
734-
"sonataflow.org/workflowName=failswitch",
735-
"-o",
736-
"wide",
737-
],
738-
120_000,
849+
banner("NetworkPolicy (can block pod-to-pod traffic to workflow Services)");
850+
dumpOc(
851+
safeOc(["get", "networkpolicy", "-n", namespace, "-o", "wide"], 60_000),
852+
"(no NetworkPolicies in namespace)",
739853
);
740-
safeOc(
741-
[
742-
"get",
743-
"pods",
744-
"-n",
745-
namespace,
746-
"-l",
747-
"sonataflow.org/workflowName=greeting",
748-
"-o",
749-
"wide",
750-
],
751-
120_000,
854+
855+
banner(
856+
"Postgres-related Secrets (existence only — compare to Deployment volume/env refs)",
857+
);
858+
banner("secret/backstage-psql-secret");
859+
dumpOc(
860+
safeOc(["get", "secret", "backstage-psql-secret", "-n", namespace], 30_000),
861+
"(get secret backstage-psql-secret — empty stdout)",
862+
);
863+
banner(
864+
"secret/sonataflow-psql-postgresql (upstream default; often absent in e2e)",
865+
);
866+
try {
867+
dumpOc(
868+
runOc(
869+
["get", "secret", "sonataflow-psql-postgresql", "-n", namespace],
870+
30_000,
871+
),
872+
"(secret absent — expected when workflows use backstage-psql-secret)",
873+
);
874+
} catch (err) {
875+
const msg = formatOcFailure(err);
876+
if (/NotFound/i.test(msg)) {
877+
console.error(
878+
"(not found — expected when workflows are patched to backstage-psql-secret)",
879+
);
880+
} else {
881+
console.error(`unexpected error: ${msg}`);
882+
}
883+
}
884+
885+
banner(
886+
"ReplicaSets in namespace (workflow — multiple RS generations / stale templates)",
752887
);
888+
dumpOc(
889+
safeOc(["get", "rs", "-n", namespace, "-o", "wide"], 120_000),
890+
"(get rs — empty stdout)",
891+
);
892+
893+
banner(
894+
"SonataFlow reconcile hints (generation / observedGeneration when present)",
895+
);
896+
for (const workflow of WORKFLOWS) {
897+
banner(`sonataflow/${workflow} (generation line)`);
898+
const gen = safeOc(
899+
[
900+
"get",
901+
"sonataflow",
902+
workflow,
903+
"-n",
904+
namespace,
905+
"-o",
906+
"jsonpath={.metadata.generation}",
907+
],
908+
30_000,
909+
);
910+
if (gen !== undefined) {
911+
console.error(` metadata.generation=${gen.trim()}`);
912+
}
913+
const observed = safeOc(
914+
[
915+
"get",
916+
"sonataflow",
917+
workflow,
918+
"-n",
919+
namespace,
920+
"-o",
921+
"jsonpath={.status.observedGeneration}",
922+
],
923+
30_000,
924+
);
925+
if (observed !== undefined && observed.trim() !== "") {
926+
console.error(` status.observedGeneration=${observed.trim()}`);
927+
}
928+
}
753929

754930
banner("recent namespace warnings/errors (last 200 events)");
755-
safeOc(
756-
["get", "events", "-n", namespace, "--sort-by=.lastTimestamp"],
757-
120_000,
931+
dumpOc(
932+
safeOc(
933+
["get", "events", "-n", namespace, "--sort-by=.lastTimestamp"],
934+
120_000,
935+
),
936+
"(get events — empty stdout)",
758937
);
759938
}
760939

0 commit comments

Comments
 (0)