From c7e0c1462435bdea7594056b6f7f55a3da1ec8fd Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Mon, 2 Mar 2026 20:34:36 +0100 Subject: [PATCH 01/11] feat: restore restart-disabled page via App CRD .spec.autoRestartEnabled When an app's CRD has autoRestartEnabled=false and the app is not in Running state, the proxy now shows the "Application Disabled" page immediately (synchronous, no DNS/wakeup needed) instead of the spinner. - AppInfo struct returned by GetState carries both ActualState and AutoRestartEnabled (pointer-with-nil defaults to true) - upstream.ServeHTTPOrError branches on AutoRestartEnabled before attempting DNS/upstream - Restores restart_disabled.go handler and restart_disabled.gohtml template (previously deleted with the Sandboxes API fallback) - Adds plain-text renderer case for Streamlit health checks - Adds setupK8s hook to testCase and a restart-disabled integration test Co-Authored-By: Claude Sonnet 4.6 --- go.mod | 17 +- go.sum | 51 ++++ .../pkg/service/appsproxy/config/config.go | 6 + .../service/appsproxy/dataapps/api/error.go | 12 - .../service/appsproxy/dataapps/api/wakeup.go | 32 --- .../appsproxy/dataapps/k8sapp/types.go | 44 ++++ .../appsproxy/dataapps/k8sapp/watcher.go | 214 ++++++++++++++++ .../appsproxy/dataapps/k8sapp/watcher_test.go | 155 +++++++++++ .../appsproxy/dataapps/wakeup/wakeup.go | 45 +--- .../appsproxy/dataapps/wakeup/wakeup_test.go | 108 ++++++-- .../appsproxy/dependencies/dependencies.go | 30 +++ .../service/appsproxy/dependencies/mocked.go | 32 ++- .../authproxy/oauthproxy/pagewriter.go | 2 +- .../proxy/apphandler/upstream/upstream.go | 46 ++-- .../appsproxy/proxy/pagewriter/error.go | 14 +- .../pkg/service/appsproxy/proxy/proxy_test.go | 240 ++++-------------- .../service/appsproxy/proxy/testutil/api.go | 19 +- 17 files changed, 727 insertions(+), 340 deletions(-) delete mode 100644 internal/pkg/service/appsproxy/dataapps/api/wakeup.go create mode 100644 internal/pkg/service/appsproxy/dataapps/k8sapp/types.go create mode 100644 internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go create mode 100644 internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go diff --git a/go.mod b/go.mod index 8a39ebefe8..35ec207eea 100644 --- a/go.mod +++ b/go.mod @@ -108,10 +108,16 @@ require ( github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect github.com/cheggaaa/pb/v3 v3.1.6 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-test/deep v1.1.1 // indirect + github.com/google/gnostic-models v0.6.9 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/icholy/gomajor v0.15.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/minio/simdjson-go v0.4.5 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/otiai10/mint v1.6.3 // indirect @@ -120,6 +126,7 @@ require ( github.com/shirou/gopsutil/v4 v4.25.8-0.20250809033336-ffcdc2b7662f // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/theckman/httpforwarded v0.4.0 // indirect + github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/etcdctl/v3 v3.6.7 // indirect go.etcd.io/gofail v0.2.0 // indirect go.etcd.io/raft/v3 v3.6.0 // indirect @@ -132,7 +139,15 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/tools/cmd/godoc v0.1.0-deprecated // indirect golang.org/x/tools/godoc v0.1.0-deprecated // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/client-go v0.33.3 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) require ( @@ -146,7 +161,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/iancoleman/strcase v0.3.0 github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 72f919e9eb..8c542ed0b4 100644 --- a/go.sum +++ b/go.sum @@ -287,6 +287,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -352,6 +353,10 @@ github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhs github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -380,8 +385,12 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/inflect v0.21.2 h1:0gClGlGcxifcJR56zwvhaOulnNgnhc4qTAkob5ObnSM= github.com/go-openapi/inflect v0.21.2/go.mod h1:INezMuUu7SJQc2AyR3WO0DqqYUJSj8Kb4hBd7WtjlAw= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-ozzo/ozzo-routing v2.1.4+incompatible h1:gQmNyAwMnBHr53Nma2gPTfVVc6i2BuAwCWPam2hIvKI= @@ -467,6 +476,10 @@ github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60 github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -516,6 +529,8 @@ github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWS github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= @@ -608,6 +623,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -636,6 +652,7 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/makeworld-the-better-one/dither/v2 v2.4.0 h1:Az/dYXiTcwcRSe59Hzw4RI1rSnAZns+1msaCXetrMFE= @@ -726,9 +743,11 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.133.0 h1:iPei+89a2EK4LuN4HeIRzZNE6XxCyrKfBKG3BkK/ViU= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.133.0/go.mod h1:asV77TgnGfc7A+a9jggdsnlLlW5dnJT8RroVuf5slko= github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.133.0 h1:4ca2pM3+xDMB9H3UnhjAiNg7EpIydZ7HdohOexU8xb8= @@ -919,6 +938,8 @@ github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIj github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28= github.com/writeas/go-strip-markdown/v2 v2.1.1/go.mod h1:UvvgPJgn1vvN8nWuE5e7v/+qmDu3BSVnKAB6Gl7hFzA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= @@ -1270,7 +1291,13 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -1298,14 +1325,38 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8= +k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY= rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= +sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= v.io/x/lib v0.1.21 h1:PtTlthjCNjjdfZviHr2hDhGSLqlyOTxSaKrBMSZOj4Q= diff --git a/internal/pkg/service/appsproxy/config/config.go b/internal/pkg/service/appsproxy/config/config.go index 7ec6ab86cb..0527e9585b 100644 --- a/internal/pkg/service/appsproxy/config/config.go +++ b/internal/pkg/service/appsproxy/config/config.go @@ -25,6 +25,7 @@ type Config struct { Upstream Upstream `configKey:"-" configUsage:"Configuration options for upstream"` SandboxesAPI SandboxesAPI `configKey:"sandboxesAPI"` CsrfTokenSalt string `configKey:"csrfTokenSalt" configUsage:"Salt used for generating CSRF tokens" validate:"required" sensitive:"true"` + K8s K8s `configKey:"k8s"` } type API struct { @@ -42,6 +43,11 @@ type Upstream struct { WsTimeout time.Duration `configKey:"wsTimeout" configUsage:"Timeout for websocket request on upstream"` } +type K8s struct { + Namespace string `configKey:"namespace" configUsage:"Kubernetes namespace containing App CRDs." validate:"required"` + Kubeconfig string `configKey:"kubeconfig" configUsage:"Path to kubeconfig file. Uses in-cluster config if empty."` +} + func New() Config { return Config{ DebugLog: false, diff --git a/internal/pkg/service/appsproxy/dataapps/api/error.go b/internal/pkg/service/appsproxy/dataapps/api/error.go index 638c9bf4cd..c57bc7ca4e 100644 --- a/internal/pkg/service/appsproxy/dataapps/api/error.go +++ b/internal/pkg/service/appsproxy/dataapps/api/error.go @@ -5,10 +5,6 @@ import ( "net/http" ) -const ( - ContextCodeRestartDisabled = "apps.restartDisabled" -) - // Error represents the structure of Sandboxes API error. type Error struct { Message string `json:"error"` @@ -37,14 +33,6 @@ func (e *Error) ErrorExceptionID() string { return e.ExceptionID } -func (e *Error) HasRestartDisabled() bool { - if e.Context == nil { - return false - } - contextCode, ok := e.Context["code"].(string) - return ok && contextCode == ContextCodeRestartDisabled -} - // StatusCode returns HTTP status code. func (e *Error) StatusCode() int { return e.response.StatusCode diff --git a/internal/pkg/service/appsproxy/dataapps/api/wakeup.go b/internal/pkg/service/appsproxy/dataapps/api/wakeup.go deleted file mode 100644 index 4d26154538..0000000000 --- a/internal/pkg/service/appsproxy/dataapps/api/wakeup.go +++ /dev/null @@ -1,32 +0,0 @@ -package api - -import ( - "context" - - "github.com/keboola/keboola-sdk-go/v2/pkg/request" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -type wakeupBody struct { - DesiredState string `json:"desiredState"` -} - -func (a *API) WakeupApp(appID AppID) request.APIRequest[request.NoResult] { - return request.NewAPIRequest(request.NoResult{}, a.newRequest(). - WithError(&Error{}). - WithOnError(func(ctx context.Context, response request.HTTPResponse, err error) error { - span := trace.SpanFromContext(ctx) - attrs := []attribute.KeyValue{ - attribute.Int(attrSandboxesServiceStatusCode, response.StatusCode()), - } - span.SetAttributes(attrs...) - return err - }). - WithPatch("apps/{appId}"). - AndPathParam("appId", appID.String()). - WithJSONBody(wakeupBody{ - DesiredState: "running", - }), - ) -} diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go new file mode 100644 index 0000000000..c78080ef6d --- /dev/null +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go @@ -0,0 +1,44 @@ +// Package k8sapp provides watching and patching of App CRDs in Kubernetes. +package k8sapp + +import "k8s.io/apimachinery/pkg/runtime/schema" + +const ( + Group = "apps.keboola.com" + Version = "v2" + Resource = "apps" +) + +// AppGVR is the GroupVersionResource for the App CRD. +var AppGVR = schema.GroupVersionResource{Group: Group, Version: Version, Resource: Resource} + +// AppActualState is the observed state of the app, read from .status.currentState. +type AppActualState string + +const ( + AppActualStateStopped AppActualState = "Stopped" + AppActualStateRunning AppActualState = "Running" + AppActualStateStarting AppActualState = "Starting" + AppActualStateStopping AppActualState = "Stopping" +) + +// appObject is a minimal struct for unmarshalling App CRD objects — only the fields we need. +type appObject struct { + Spec appSpec `json:"spec"` + Status appStatus `json:"status"` +} + +type appSpec struct { + AppID string `json:"appId"` + AutoRestartEnabled *bool `json:"autoRestartEnabled,omitempty"` +} + +// AppInfo is the cached state for an app, read from the K8s watcher. +type AppInfo struct { + ActualState AppActualState + AutoRestartEnabled bool +} + +type appStatus struct { + CurrentState AppActualState `json:"currentState"` +} diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go new file mode 100644 index 0000000000..61bd205139 --- /dev/null +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go @@ -0,0 +1,214 @@ +package k8sapp + +import ( + "context" + "encoding/json" + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + + "github.com/keboola/keboola-as-code/internal/pkg/log" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" +) + +// entry stores the K8s object name and last observed state for an app. +type entry struct { + k8sName string + state AppActualState + autoRestartEnabled bool +} + +// StateWatcher watches App CRDs in Kubernetes and provides a local cache of app states. +type StateWatcher struct { + client dynamic.Interface + namespace string + logger log.Logger + // byName: K8s object name → AppID + byName sync.Map + // byAppID: AppID → entry{k8sName, state} + byAppID sync.Map +} + +type dependencies interface { + Logger() log.Logger + Process() *servicectx.Process +} + +// NewDynamicClient creates a Kubernetes dynamic client from kubeconfig path or in-cluster config. +// If kubeconfigPath is empty, in-cluster config is used. +func NewDynamicClient(kubeconfigPath string) (dynamic.Interface, error) { + var cfg *rest.Config + var err error + if kubeconfigPath != "" { + cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath) + } else { + cfg, err = rest.InClusterConfig() + } + if err != nil { + return nil, err + } + return dynamic.NewForConfig(cfg) +} + +// NewStateWatcher creates and starts a StateWatcher that watches App CRDs in the given namespace. +// It registers the informer lifecycle with the process. +func NewStateWatcher(d dependencies, client dynamic.Interface, namespace string) *StateWatcher { + w := &StateWatcher{ + client: client, + namespace: namespace, + logger: d.Logger().WithComponent("k8sapp.watcher"), + } + + ctx, cancel := context.WithCancel(context.Background()) + d.Process().OnShutdown(func(context.Context) { + cancel() + }) + + lw := &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return client.Resource(AppGVR).Namespace(namespace).List(ctx, opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return client.Resource(AppGVR).Namespace(namespace).Watch(ctx, opts) + }, + } + + informer := cache.NewSharedIndexInformer( + lw, + &unstructured.Unstructured{}, + // No resync — rely on watch events only. + 0, + cache.Indexers{}, + ) + + _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + w.handleUpsert(ctx, obj) + }, + UpdateFunc: func(_, newObj any) { + w.handleUpsert(ctx, newObj) + }, + DeleteFunc: func(obj any) { + w.handleDelete(ctx, obj) + }, + }) + if err != nil { + w.logger.Errorf(ctx, "failed to add event handler to App informer: %s", err) + } + + go informer.Run(ctx.Done()) + + // Log when the cache has synced so operators know the watcher is ready. + go func() { + if cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) { + w.logger.Infof(ctx, "App CRD cache synced for namespace %q", namespace) + } + }() + + return w +} + +// GetState returns the cached AppInfo for the app. Returns (AppInfo{}, false) if not yet cached. +func (w *StateWatcher) GetState(appID api.AppID) (AppInfo, bool) { + v, ok := w.byAppID.Load(appID) + if !ok { + return AppInfo{}, false + } + e := v.(entry) + return AppInfo{ActualState: e.state, AutoRestartEnabled: e.autoRestartEnabled}, true +} + +// SetDesiredRunning patches .spec.state = "Running" on the App CRD for the given appID. +// If the appID is not yet in the cache, no patch is sent. +func (w *StateWatcher) SetDesiredRunning(ctx context.Context, appID api.AppID) error { + v, ok := w.byAppID.Load(appID) + if !ok { + return nil + } + e := v.(entry) + + patch, err := json.Marshal(map[string]any{ + "spec": map[string]any{ + "state": "Running", + }, + }) + if err != nil { + return err + } + + _, err = w.client.Resource(AppGVR).Namespace(w.namespace).Patch( + ctx, + e.k8sName, + k8stypes.MergePatchType, + patch, + metav1.PatchOptions{}, + ) + return err +} + +func (w *StateWatcher) handleUpsert(ctx context.Context, obj any) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return + } + k8sName := u.GetName() + + data, err := u.MarshalJSON() + if err != nil { + w.logger.Errorf(ctx, "failed to marshal App CRD %q: %s", k8sName, err) + return + } + + var appObj appObject + if err = json.Unmarshal(data, &appObj); err != nil { + w.logger.Errorf(ctx, "failed to unmarshal App CRD %q: %s", k8sName, err) + return + } + + if appObj.Spec.AppID == "" { + w.logger.Warnf(ctx, "App CRD %q has empty spec.appId, skipping", k8sName) + return + } + + autoRestartEnabled := true + if appObj.Spec.AutoRestartEnabled != nil { + autoRestartEnabled = *appObj.Spec.AutoRestartEnabled + } + + appID := api.AppID(appObj.Spec.AppID) + w.byName.Store(k8sName, appID) + w.byAppID.Store(appID, entry{k8sName: k8sName, state: appObj.Status.CurrentState, autoRestartEnabled: autoRestartEnabled}) +} + +func (w *StateWatcher) handleDelete(ctx context.Context, obj any) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + // Handle tombstone objects from the cache. + tombstone, ok2 := obj.(cache.DeletedFinalStateUnknown) + if !ok2 { + return + } + u, ok = tombstone.Obj.(*unstructured.Unstructured) + if !ok { + return + } + } + k8sName := u.GetName() + + v, loaded := w.byName.LoadAndDelete(k8sName) + if !loaded { + return + } + appID := v.(api.AppID) + w.byAppID.Delete(appID) + w.logger.Debugf(ctx, "App CRD %q (appID=%s) removed from cache", k8sName, appID) +} diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go new file mode 100644 index 0000000000..6d40a147b7 --- /dev/null +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go @@ -0,0 +1,155 @@ +package k8sapp_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/keboola/keboola-as-code/internal/pkg/log" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" + "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" +) + +const testNamespace = "keboola" + +// watcherDeps implements k8sapp.StateWatcher dependencies for tests. +type watcherDeps struct { + logger log.Logger + proc *servicectx.Process +} + +func newTestDeps(t *testing.T) *watcherDeps { + t.Helper() + logger := log.NewNopLogger() + proc := servicectx.New(servicectx.WithLogger(logger), servicectx.WithoutSignals()) + t.Cleanup(func() { + proc.Shutdown(context.Background(), nil) + proc.WaitForShutdown() + }) + return &watcherDeps{logger: logger, proc: proc} +} + +func (d *watcherDeps) Logger() log.Logger { return d.logger } +func (d *watcherDeps) Process() *servicectx.Process { return d.proc } + +// newFakeClient creates a fake dynamic client with the App list kind registered. +func newFakeClient(objects ...runtime.Object) *k8sfake.FakeDynamicClient { + scheme := runtime.NewScheme() + return k8sfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + k8sapp.AppGVR: "AppList", + }, objects...) +} + +// newAppObject creates an unstructured App CRD object. +func newAppObject(k8sName, appID string, state k8sapp.AppActualState) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": k8sapp.Group + "/" + k8sapp.Version, + "kind": "App", + "metadata": map[string]any{ + "name": k8sName, + "namespace": testNamespace, + }, + "spec": map[string]any{ + "appId": appID, + }, + "status": map[string]any{ + "currentState": string(state), + }, + }, + } +} + +func TestStateWatcher_GetState_UnknownWhenEmpty(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + watcher := k8sapp.NewStateWatcher(newTestDeps(t), fakeClient, testNamespace) + + info, ok := watcher.GetState(api.AppID("app-123")) + assert.False(t, ok) + assert.Empty(t, info.ActualState) +} + +func TestStateWatcher_GetState_AfterCacheSync(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + // Add the App CRD object to the fake tracker before the watcher starts. + appObj := newAppObject("my-app-k8s", "app-123", k8sapp.AppActualStateStopped) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + assert.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("app-123")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) +} + +func TestStateWatcher_SetDesiredRunning(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + appObj := newAppObject("my-app-k8s", "app-123", k8sapp.AppActualStateStopped) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + // Wait for the informer to cache the object. + require.Eventually(t, func() bool { + _, ok := watcher.GetState(api.AppID("app-123")) + return ok + }, 5*time.Second, 50*time.Millisecond) + + // Clear prior actions (list/watch from informer startup). + fakeClient.ClearActions() + + err = watcher.SetDesiredRunning(t.Context(), api.AppID("app-123")) + require.NoError(t, err) + + // Verify that a merge-patch action targeting App CRDs was recorded. + actions := fakeClient.Actions() + require.Len(t, actions, 1) + + pa, ok := actions[0].(k8stesting.PatchAction) + require.True(t, ok, "expected a PatchAction") + assert.Equal(t, k8stypes.MergePatchType, pa.GetPatchType()) + assert.Contains(t, string(pa.GetPatch()), `"state":"Running"`) +} + +func TestStateWatcher_SetDesiredRunning_NoOpWhenUnknown(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + watcher := k8sapp.NewStateWatcher(newTestDeps(t), fakeClient, testNamespace) + + // App not in K8s cache — SetDesiredRunning should be a no-op. + err := watcher.SetDesiredRunning(t.Context(), api.AppID("app-unknown")) + require.NoError(t, err) + + for _, a := range fakeClient.Actions() { + assert.NotEqual(t, "patch", a.GetVerb(), "unexpected PATCH for unknown app") + } +} diff --git a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go index 3ce6f057c4..1e4d182c05 100644 --- a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go +++ b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go @@ -11,21 +11,18 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/syncmap" - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" ) -// Interval sets how often the proxy sends wakeup request to sandboxes service. -// If the last notification for the app was less than this Interval ago then the notification is skipped. -const ( - Interval = time.Second - wakeupErrorToBeSkipped = `can't have desired state "running". Currently is in state: "stopping", desired state: "stopped"` -) +// Interval sets how often the proxy sends a wakeup request. +// If the last wakeup for the app was less than this Interval ago then the wakeup is skipped. +const Interval = time.Second type Manager struct { clock clockwork.Clock logger log.Logger - api *api.API + watcher *k8sapp.StateWatcher stateMap *syncmap.SyncMap[api.AppID, state] } @@ -37,14 +34,14 @@ type state struct { type dependencies interface { Clock() clockwork.Clock Logger() log.Logger - AppsAPI() *api.API + AppStateWatcher() *k8sapp.StateWatcher } func NewManager(d dependencies) *Manager { return &Manager{ - clock: d.Clock(), - logger: d.Logger(), - api: d.AppsAPI(), + clock: d.Clock(), + logger: d.Logger(), + watcher: d.AppStateWatcher(), stateMap: syncmap.New[api.AppID, state](func(api.AppID) *state { return &state{lock: &sync.Mutex{}} }), @@ -55,38 +52,22 @@ func (l *Manager) Wakeup(ctx context.Context, appID api.AppID) error { // Get cache item or init an empty item item := l.stateMap.GetOrInit(appID) - // Only one notification runs in parallel. + // Only one wakeup runs in parallel. // If there is an in-flight update, we are waiting for its results. item.lock.Lock() defer item.lock.Unlock() - // Return config from cache if still valid now := l.clock.Now() - if now.Before(item.nextRequestAfter) { - // Skip if a notification was sent less than Interval ago + // Skip if a wakeup was sent less than Interval ago return nil } // Update nextRequestAfter time item.nextRequestAfter = now.Add(Interval) - // Send the notification - _, err := l.api.WakeupApp(appID).Send(ctx) - - // Check if it's a restart disabled error via context code - var apiErr *api.Error - if errors.As(err, &apiErr) && apiErr.HasRestartDisabled() { - // This is expected for apps with restart disabled, don't log as error - l.logger.Infof(ctx, `app "%s" has restart disabled`, appID) - return err // Still return the error so it can be handled by the proxy - } - - // If it does not succeed but app is currently stopping do not log it as error, log only other errors - // Instead of implementing state machine as in sandboxes service, we want to skip valid state that the - // pod is deallocating, and we want to wait till pod is `stopped` and we can `start` the pod again. - if err != nil && err.Error() != wakeupErrorToBeSkipped { - l.logger.Errorf(ctx, `failed sending wakeup request to Sandboxes Service about for app "%s": %s`, appID, err.Error()) + if err := l.watcher.SetDesiredRunning(ctx, appID); err != nil { + l.logger.Errorf(ctx, `failed setting desired state "Running" for app "%s": %s`, appID, err) return err } return nil diff --git a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go index 869bf1954e..09bb118138 100644 --- a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go +++ b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go @@ -2,25 +2,57 @@ package wakeup_test import ( "context" - "fmt" - "net/http" "sync" "testing" "time" - "github.com/jarcoal/httpmock" "github.com/jonboulle/clockwork" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/atomic" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8stesting "k8s.io/client-go/testing" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/wakeup" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dependencies" commonDeps "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" ) +const testNamespace = "keboola" + +func newTestApp(appID string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": k8sapp.Group + "/" + k8sapp.Version, + "kind": "App", + "metadata": map[string]any{ + "name": "app-k8s-" + appID, + "namespace": testNamespace, + }, + "spec": map[string]any{ + "appId": appID, + }, + "status": map[string]any{ + "currentState": string(k8sapp.AppActualStateStopped), + }, + }, + } +} + +func patchCount(actions []k8stesting.Action) int { + count := 0 + for _, a := range actions { + if a.GetVerb() == "patch" { + count++ + } + } + return count +} + func TestManager_Wakeup(t *testing.T) { t.Parallel() @@ -29,32 +61,44 @@ func TestManager_Wakeup(t *testing.T) { d, mock := dependencies.NewMockedServiceScope(t, ctx, config.New(), commonDeps.WithClock(clk)) appID := api.AppID("app") + fakeClient := mock.TestFakeK8sClient() - transport := mock.MockedHTTPTransport() - transport.RegisterResponder( - http.MethodPatch, - fmt.Sprintf("%s/apps/%s", mock.TestConfig().SandboxesAPI.URL, appID), - httpmock.NewStringResponder(http.StatusOK, ""), + // Register app in fake K8s so SetDesiredRunning has a target. + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + ctx, newTestApp(string(appID)), metav1.CreateOptions{}, ) + require.NoError(t, err) manager := d.WakeupManager() + watcher := d.AppStateWatcher() - // The first request is send to the API - err := manager.Wakeup(ctx, appID) + // Wait for watcher cache to sync. + require.Eventually(t, func() bool { + _, ok := watcher.GetState(appID) + return ok + }, 5*time.Second, 50*time.Millisecond) + + // Clear list/watch actions from informer startup. + fakeClient.ClearActions() + + // The first wakeup sends a K8s PATCH. + err = manager.Wakeup(ctx, appID) require.NoError(t, err) - assert.Equal(t, 1, transport.GetTotalCallCount()) + assert.Equal(t, 1, patchCount(fakeClient.Actions())) - // Request is skipped, the Interval was not exceeded + // Second call is skipped because the Interval was not exceeded. + fakeClient.ClearActions() clk.Advance(time.Millisecond) err = manager.Wakeup(ctx, appID) require.NoError(t, err) - assert.Equal(t, 1, transport.GetTotalCallCount()) + assert.Equal(t, 0, patchCount(fakeClient.Actions())) - // Exceed the Interval + // After exceeding the Interval, a new PATCH is sent. + fakeClient.ClearActions() clk.Advance(wakeup.Interval) err = manager.Wakeup(ctx, appID) require.NoError(t, err) - assert.Equal(t, 2, transport.GetTotalCallCount()) + assert.Equal(t, 1, patchCount(fakeClient.Actions())) } func TestManager_Wakeup_Race(t *testing.T) { @@ -65,35 +109,43 @@ func TestManager_Wakeup_Race(t *testing.T) { d, mock := dependencies.NewMockedServiceScope(t, ctx, config.New(), commonDeps.WithClock(clk)) appID := api.AppID("app") + fakeClient := mock.TestFakeK8sClient() - transport := mock.MockedHTTPTransport() - transport.RegisterResponder( - http.MethodPatch, - fmt.Sprintf("%s/apps/%s", mock.TestConfig().SandboxesAPI.URL, appID), - httpmock.NewStringResponder(http.StatusOK, ""), + // Register app in fake K8s. + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + ctx, newTestApp(string(appID)), metav1.CreateOptions{}, ) + require.NoError(t, err) manager := d.WakeupManager() + watcher := d.AppStateWatcher() - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + // Wait for watcher cache to sync. + require.Eventually(t, func() bool { + _, ok := watcher.GetState(appID) + return ok + }, 5*time.Second, 50*time.Millisecond) + + // Clear list/watch actions from informer startup. + fakeClient.ClearActions() + + raceCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() wg := sync.WaitGroup{} counter := atomic.NewInt64(0) - // Load configuration 10x in parallel + // Call Wakeup 10x in parallel. for range 10 { wg.Go(func() { - err := manager.Wakeup(ctx, appID) - require.NoError(t, err) - + require.NoError(t, manager.Wakeup(raceCtx, appID)) counter.Add(1) }) } - // Wait for all requests wg.Wait() - // Check total goroutines/requests count + // All goroutines completed. assert.Equal(t, int64(10), counter.Load()) - assert.Equal(t, 1, transport.GetTotalCallCount()) + // Only one K8s PATCH is sent due to rate-limiting. + assert.Equal(t, 1, patchCount(fakeClient.Actions())) } diff --git a/internal/pkg/service/appsproxy/dependencies/dependencies.go b/internal/pkg/service/appsproxy/dependencies/dependencies.go index 60fba1542b..c28ff09227 100644 --- a/internal/pkg/service/appsproxy/dependencies/dependencies.go +++ b/internal/pkg/service/appsproxy/dependencies/dependencies.go @@ -20,11 +20,14 @@ import ( "net/http" "github.com/jonboulle/clockwork" + "k8s.io/client-go/dynamic" + k8sfake "k8s.io/client-go/dynamic/fake" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/appconfig" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/notify" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/wakeup" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler" @@ -79,12 +82,14 @@ type ServiceScope interface { PageWriter() *pagewriter.Writer NotifyManager() *notify.Manager WakeupManager() *wakeup.Manager + AppStateWatcher() *k8sapp.StateWatcher } type Mocked interface { dependencies.Mocked TestConfig() config.Config TestDNSServer() *dnsmock.Server + TestFakeK8sClient() *k8sfake.FakeDynamicClient } // serviceScope implements APIScope interface. @@ -100,12 +105,19 @@ type serviceScope struct { appConfigLoader appconfig.Loader notifyManager *notify.Manager wakeupManager *wakeup.Manager + appStateWatcher *k8sapp.StateWatcher } type parentScopes interface { dependencies.BaseScope } +// k8sClientProvider is optionally implemented by parentScopes to supply an already-constructed +// dynamic client (used in tests to inject a fake client without real kubeconfig). +type k8sClientProvider interface { + K8sDynamicClient() dynamic.Interface +} + type parentScopesImpl struct { dependencies.BaseScope } @@ -185,6 +197,20 @@ func newServiceScope(ctx context.Context, parentScp parentScopes, cfg config.Con d.appsAPI = api.New(d.HTTPClient(), cfg.SandboxesAPI.URL, cfg.SandboxesAPI.Token) d.appConfigLoader = appconfig.NewLoader(d) d.notifyManager = notify.NewManager(d) + + if cfg.K8s.Namespace != "" { + var k8sClient dynamic.Interface + if provider, ok := parentScp.(k8sClientProvider); ok { + k8sClient = provider.K8sDynamicClient() + } else { + k8sClient, err = k8sapp.NewDynamicClient(cfg.K8s.Kubeconfig) + if err != nil { + return nil, err + } + } + d.appStateWatcher = k8sapp.NewStateWatcher(d, k8sClient, cfg.K8s.Namespace) + } + d.wakeupManager = wakeup.NewManager(d) d.authProxyManager = authproxy.NewManager(d) d.upstreamManager = upstream.NewManager(d) @@ -232,3 +258,7 @@ func (v *serviceScope) NotifyManager() *notify.Manager { func (v *serviceScope) WakeupManager() *wakeup.Manager { return v.wakeupManager } + +func (v *serviceScope) AppStateWatcher() *k8sapp.StateWatcher { + return v.appStateWatcher +} diff --git a/internal/pkg/service/appsproxy/dependencies/mocked.go b/internal/pkg/service/appsproxy/dependencies/mocked.go index 21edb20bd7..ee7c71fa85 100644 --- a/internal/pkg/service/appsproxy/dependencies/mocked.go +++ b/internal/pkg/service/appsproxy/dependencies/mocked.go @@ -6,8 +6,13 @@ import ( "testing" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + k8sfake "k8s.io/client-go/dynamic/fake" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" @@ -16,8 +21,9 @@ import ( // mocked implements Mocked interface. type mocked struct { dependencies.Mocked - config config.Config - dnsServer *dnsmock.Server + config config.Config + dnsServer *dnsmock.Server + fakeK8sClient *k8sfake.FakeDynamicClient } func (v *mocked) TestConfig() config.Config { @@ -28,6 +34,17 @@ func (v *mocked) TestDNSServer() *dnsmock.Server { return v.dnsServer } +// TestFakeK8sClient returns the fake Kubernetes dynamic client used by this mock. +// Tests can use it to pre-populate App CRD objects and inspect PATCH actions. +func (v *mocked) TestFakeK8sClient() *k8sfake.FakeDynamicClient { + return v.fakeK8sClient +} + +// K8sDynamicClient implements k8sClientProvider, supplying the fake client to newServiceScope. +func (v *mocked) K8sDynamicClient() dynamic.Interface { + return v.fakeK8sClient +} + func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config, opts ...dependencies.MockedOption) (ServiceScope, Mocked) { tb.Helper() @@ -51,6 +68,9 @@ func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config if cfg.SandboxesAPI.Token == "" { cfg.SandboxesAPI.Token = "my-token" } + if cfg.K8s.Namespace == "" { + cfg.K8s.Namespace = "keboola" + } var dnsServer *dnsmock.Server if cfg.DNSServer == "" { @@ -63,10 +83,16 @@ func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config cfg.DNSServer = dnsServer.Addr() } + // Create fake K8s dynamic client. The App list kind is registered so the informer can list CRDs. + scheme := runtime.NewScheme() + fakeClient := k8sfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + k8sapp.AppGVR: "AppList", + }) + // Validate config require.NoError(tb, configmap.ValidateAndNormalize(&cfg)) - mock := &mocked{Mocked: commonMock, config: cfg, dnsServer: dnsServer} + mock := &mocked{Mocked: commonMock, config: cfg, dnsServer: dnsServer, fakeK8sClient: fakeClient} scope, err := newServiceScope(ctx, mock, cfg) require.NoError(tb, err) diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go b/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go index 014ca7b0a9..142277a603 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go @@ -99,7 +99,7 @@ func (pw *pageWriter) WriteErrorPage(w http.ResponseWriter, req *http.Request, o } func (pw *pageWriter) ProxyErrorHandler(w http.ResponseWriter, req *http.Request, err error) { - pw.pageWriter.ProxyErrorHandler(w, req, pw.app, nil, err) + pw.pageWriter.ProxyErrorHandler(w, req, pw.app, err) } func (pw *pageWriter) WriteRobotsTxt(w http.ResponseWriter, req *http.Request) { diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index a00f52065e..7322f43028 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -18,6 +18,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/appconfig" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/notify" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/wakeup" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/chain" @@ -45,18 +46,18 @@ type Manager struct { configLoader appconfig.Loader notify *notify.Manager wakeup *wakeup.Manager + stateWatcher *k8sapp.StateWatcher config config.Config } type AppUpstream struct { - manager *Manager - app api.AppConfig - target *url.URL - handler *chain.Chain - wsHandler *chain.Chain - cancelWs context.CancelCauseFunc - activeWsCount atomic.Int64 - restartDisabled atomic.Bool + manager *Manager + app api.AppConfig + target *url.URL + handler *chain.Chain + wsHandler *chain.Chain + cancelWs context.CancelCauseFunc + activeWsCount atomic.Int64 } type dependencies interface { @@ -68,6 +69,7 @@ type dependencies interface { AppConfigLoader() appconfig.Loader NotifyManager() *notify.Manager WakeupManager() *wakeup.Manager + AppStateWatcher() *k8sapp.StateWatcher Config() config.Config } @@ -81,6 +83,7 @@ func NewManager(d dependencies) *Manager { configLoader: d.AppConfigLoader(), notify: d.NotifyManager(), wakeup: d.WakeupManager(), + stateWatcher: d.AppStateWatcher(), config: d.Config(), } @@ -132,6 +135,20 @@ func (m *Manager) NewUpstream(ctx context.Context, app api.AppConfig) (upstream } func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request) error { + // K8s state pre-check: if we know the app is not running, handle it synchronously + // without attempting DNS/upstream. Falls through if state is unknown or Running. + if u.manager.stateWatcher != nil { + if appInfo, ok := u.manager.stateWatcher.GetState(u.app.ID); ok && appInfo.ActualState != k8sapp.AppActualStateRunning { + if !appInfo.AutoRestartEnabled { + u.manager.pageWriter.WriteRestartDisabledPage(rw, req, u.app) + return nil + } + u.wakeup(req.Context(), errors.Errorf("app state is %s", appInfo.ActualState)) + u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + return nil + } + } + // Difference between regular and websocket request if strings.EqualFold(req.Header.Get("Connection"), "upgrade") && req.Header.Get("Upgrade") == "websocket" { return u.wsHandler.ServeHTTPOrError(rw, req) @@ -142,7 +159,7 @@ func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { proxy := httputil.NewSingleHostReverseProxy(u.target) proxy.Transport = u.manager.transport - proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app, &u.restartDisabled) + proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app) return chain. New(chain.HandlerFunc(func(w http.ResponseWriter, req *http.Request) error { @@ -161,7 +178,7 @@ func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { func (u *AppUpstream) newWebsocketProxy(timeout time.Duration) *chain.Chain { proxy := httputil.NewSingleHostReverseProxy(u.target) proxy.Transport = u.manager.transport - proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app, &u.restartDisabled) + proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app) return chain. New(chain.HandlerFunc(func(w http.ResponseWriter, req *http.Request) error { @@ -197,8 +214,6 @@ func (u *AppUpstream) trace() chain.Middleware { DNSDone: func(info httptrace.DNSDoneInfo) { if info.Err != nil { u.wakeup(ctx, info.Err) - } else { - u.restartDisabled.Store(false) } }, }) @@ -235,13 +250,6 @@ func (u *AppUpstream) wakeup(ctx context.Context, err error) { // Error is already logged by the Wakeup method itself. err := u.manager.wakeup.Wakeup(wakeupCtx, u.app.ID) //nolint:contextcheck - - // Check for restart disabled error - var apiErr *api.Error - if errors.As(err, &apiErr) && apiErr.HasRestartDisabled() { - u.restartDisabled.Store(true) - } - span.End(&err) }) } diff --git a/internal/pkg/service/appsproxy/proxy/pagewriter/error.go b/internal/pkg/service/appsproxy/proxy/pagewriter/error.go index 39a10a5137..27e603b9ff 100644 --- a/internal/pkg/service/appsproxy/proxy/pagewriter/error.go +++ b/internal/pkg/service/appsproxy/proxy/pagewriter/error.go @@ -4,7 +4,6 @@ import ( "net" "net/http" "strings" - "sync/atomic" "go.opentelemetry.io/otel/attribute" semconv "go.opentelemetry.io/otel/semconv/v1.17.0" @@ -27,20 +26,13 @@ type errorPageData struct { ExceptionID string } -func (pw *Writer) ProxyErrorHandlerFor(app api.AppConfig, restartDisabled *atomic.Bool) func(w http.ResponseWriter, req *http.Request, err error) { +func (pw *Writer) ProxyErrorHandlerFor(app api.AppConfig) func(w http.ResponseWriter, req *http.Request, err error) { return func(w http.ResponseWriter, req *http.Request, err error) { - pw.ProxyErrorHandler(w, req, app, restartDisabled, err) + pw.ProxyErrorHandler(w, req, app, err) } } -func (pw *Writer) ProxyErrorHandler(w http.ResponseWriter, req *http.Request, app api.AppConfig, restartDisabled *atomic.Bool, err error) { - // Check for restart disabled error - if restartDisabled.Load() { - pw.logger.Info(req.Context(), "app has restart disabled, rendering restart disabled page") - pw.WriteRestartDisabledPage(w, req, app) - return - } - +func (pw *Writer) ProxyErrorHandler(w http.ResponseWriter, req *http.Request, app api.AppConfig, err error) { var dnsError *net.DNSError if errors.As(err, &dnsError) { pw.logger.Info(req.Context(), "app is not running, rendering spinner page") diff --git a/internal/pkg/service/appsproxy/proxy/proxy_test.go b/internal/pkg/service/appsproxy/proxy/proxy_test.go index 44ac508a80..269114286e 100644 --- a/internal/pkg/service/appsproxy/proxy/proxy_test.go +++ b/internal/pkg/service/appsproxy/proxy/proxy_test.go @@ -36,10 +36,15 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8sfake "k8s.io/client-go/dynamic/fake" + "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/auth/provider" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" proxyDependencies "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/logging" @@ -54,9 +59,9 @@ import ( type testCase struct { name string + setupK8s func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) run func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) expectedNotifications map[string]int - expectedWakeUps map[string]int expectedSpans tracetest.SpanStubs } @@ -78,7 +83,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, "OK\n", string(body)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "missing-app-id", @@ -94,7 +98,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Unexpected domain, missing application ID.") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "unknown-app-id", @@ -110,7 +113,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), html.EscapeString(`Application "unknown" not found in the stack.`)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "broken-app", @@ -127,7 +129,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, expectedSpans: []tracetest.SpanStub{ { Name: "keboola.go.common.dependencies.NewBaseScope", @@ -232,7 +233,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "wrong-rule-type-app", @@ -249,7 +249,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "missing-referenced-provider", @@ -266,7 +265,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "empty-allowed-roles-array-app", @@ -283,7 +281,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "unknown-provider-app", @@ -300,7 +297,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "not-empty-providers-auth-required-false", @@ -317,7 +313,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "redirect-to-canonical-host", @@ -332,7 +327,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, "https://public-123.hub.keboola.local/some/data/app/url?foo=bar", location) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "redirect-to-host-lowercase", @@ -346,7 +340,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, "https://lowercase-12345.hub.keboola.local/some/data/app/url?foo=bar", location) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-down", @@ -364,7 +357,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), `Request to application failed.`) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-sub-url", @@ -395,7 +387,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "private-app-verified-email", @@ -482,7 +473,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "private-app-unauthorized", @@ -517,7 +507,6 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-missing-csrf-token", @@ -588,7 +577,6 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-group-mismatch", @@ -643,7 +631,6 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-unverified-email", @@ -696,7 +683,6 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-oidc-down", @@ -748,7 +734,6 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-down", @@ -840,7 +825,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), `Request to application failed.`) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-basic-flow", @@ -946,7 +930,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "multi": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-redirect-to-selection-page", @@ -978,7 +961,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), htmlLinkTo(`https://multi.hub.keboola.local/_proxy/selection?provider=oidc2`)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-unverified-email", @@ -1047,7 +1029,6 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusUnauthorized, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-down", @@ -1125,7 +1106,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), `Request to application failed.`) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-broken-provider", @@ -1153,7 +1133,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-websocket", @@ -1181,7 +1160,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "private-app-websocket-unauthorized", @@ -1200,7 +1178,6 @@ func TestAppProxyRouter(t *testing.T) { require.Contains(t, err.Error(), "failed to WebSocket dial: expected handshake response status code 101 but got 302") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-websocket", @@ -1273,7 +1250,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "websocket-connection-check", @@ -1304,7 +1280,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "111": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-websocket", @@ -1397,7 +1372,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "multi": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "prefix-app-no-auth", @@ -1433,7 +1407,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "prefix-app-api-auth", @@ -1496,7 +1469,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "prefix-app-web-auth", @@ -1578,7 +1550,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "shared-provider", @@ -1666,7 +1637,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "configuration-change", @@ -1713,7 +1683,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "concurrency-test", @@ -1750,7 +1719,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, int64(100), counter.Load()) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-wakeup", @@ -1779,9 +1747,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{ - "123": 1, - }, }, { name: "public-app-wakeup-only", @@ -1801,135 +1766,8 @@ func TestAppProxyRouter(t *testing.T) { // Expect wakeup but no notification since there was an authorized request to the app but not while it was running. }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{ - "123": 1, - }, }, - { - name: "restart-disabled", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - service.WakeUpOverrides["123"] = func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusBadRequest) - _, _ = fmt.Fprintln(w, `{ - "code": 0, - "context": { - "code": "apps.restartDisabled" - }, - "error": "App restart is disabled. Contact app maintainer.", - "exceptionId": "exception-208db995c92ed365d47bcc701ae4d802", - "status": "error" -}`) - } - // Request to public app - fails because the app doesn't have a DNS record - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - // First request returns 503 Service Unavailable (Spinner) because wakeup is async - require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - body, err := io.ReadAll(response.Body) - require.NoError(t, err) - assert.Contains(t, string(body), "Starting your application...") - - // Wait for async wakeup to complete - assert.Eventually(t, func() bool { - // Second request should return 404 Not Found (Restart Disabled) - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - if response.StatusCode == http.StatusServiceUnavailable { - body, err := io.ReadAll(response.Body) - require.NoError(t, err) - return strings.Contains(string(body), "Application Disabled") - } - return false - }, 5*time.Second, 100*time.Millisecond) - - // Expect wakeup but no notification since there was an authorized request to the app but not while it was running. - }, - expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, - }, - { - name: "restart-disabled-flag-reset-on-dns-success", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - // Phase 1: Set restart disabled state - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - - service.WakeUpOverrides["123"] = func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusBadRequest) - _, _ = fmt.Fprintln(w, `{ - "code": 0, - "context": { - "code": "apps.restartDisabled" - }, - "error": "App restart is disabled. Contact app maintainer.", - "exceptionId": "exception-208db995c92ed365d47bcc701ae4d802", - "status": "error" -}`) - } - - // Request to public app - fails because the app doesn't have a DNS record - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - // First request returns 503 Service Unavailable (Spinner) because wakeup is async - require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - - // Wait for async wakeup to complete and flag to be set - assert.Eventually(t, func() bool { - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - body, err := io.ReadAll(response.Body) - require.NoError(t, err) - return response.StatusCode == http.StatusServiceUnavailable && strings.Contains(string(body), "Application Disabled") - }, 5*time.Second, 100*time.Millisecond) - - // Phase 2: Simulate app becoming available (restart enabled) - // Add DNS record back and remove wakeup override - dnsServer.AddARecord(dns.Fqdn("app.local"), net.ParseIP("127.0.0.1")) - - // Wait for DNS to be fully propagated before removing override - assert.Eventually(t, func() bool { - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - _, _ = io.ReadAll(response.Body) - return response.StatusCode == http.StatusOK - }, 5*time.Second, 100*time.Millisecond) - - // Now safe to remove override - delete(service.WakeUpOverrides, "123") - - // Request should now succeed - DNS resolution succeeds, flag gets reset - request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err = client.Do(request) - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - body, err := io.ReadAll(response.Body) - require.NoError(t, err) - assert.Contains(t, string(body), "Hello, client") - - // Verify flag stays reset - subsequent requests should continue to work - request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err = client.Do(request) - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - }, - expectedNotifications: map[string]int{"123": 1}, - expectedWakeUps: map[string]int{}, - }, { name: "private-one-provider-selector", run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { @@ -1944,7 +1782,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), htmlLinkTo(`https://oidc.hub.keboola.local/_proxy/selection?provider=oidc`)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-wakeup", @@ -2016,9 +1853,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{ - "oidc": 1, - }, }, { name: "private-app-wakeup-only", @@ -2084,9 +1918,6 @@ func TestAppProxyRouter(t *testing.T) { // Expect wakeup but no notification since there was an authorized request to the app but not while it was running. }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{ - "oidc": 1, - }, }, { name: "private-app-no-wakeup", @@ -2142,7 +1973,6 @@ func TestAppProxyRouter(t *testing.T) { // Expect no notification or wakeup because there was never an authorized request to the app }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-wrong-login-no-password", @@ -2169,7 +1999,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Please enter a correct password.") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-wrong-login", @@ -2196,7 +2025,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Please enter a correct password.") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-correct-app-url", @@ -2244,7 +2072,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "auth": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-correct-login", @@ -2295,7 +2122,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "auth": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-wrong-cookie", @@ -2312,7 +2138,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Cookie has expired") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-cookie", @@ -2332,7 +2157,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "auth": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-sign-out", @@ -2358,7 +2182,50 @@ func TestAppProxyRouter(t *testing.T) { require.Empty(t, response.Cookies()) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, + }, + { + name: "restart-disabled", + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + autoRestartEnabled := false + appObj := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": k8sapp.Group + "/" + k8sapp.Version, + "kind": "App", + "metadata": map[string]any{ + "name": "app-123", + "namespace": "keboola", + }, + "spec": map[string]any{ + "appId": "123", + "autoRestartEnabled": autoRestartEnabled, + }, + "status": map[string]any{ + "currentState": string(k8sapp.AppActualStateStopped), + }, + }, + } + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && !info.AutoRestartEnabled + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + // Use canonical slug-based URL to bypass the slug redirect. + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) + require.NoError(t, err) + response, err := client.Do(request) + require.NoError(t, err) + require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + assert.Contains(t, string(body), "Application Disabled") + }, + expectedNotifications: map[string]int{}, }, } @@ -2379,7 +2246,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, } } @@ -2485,7 +2351,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{}, } } @@ -2561,13 +2426,16 @@ func TestAppProxyRouter(t *testing.T) { client := createHTTPClient(t, proxyURL) + if tc.setupK8s != nil { + tc.setupK8s(t, mocked.TestFakeK8sClient(), d.AppStateWatcher()) + } + tc.run(t, client, providers, appServer, appsAPI, dnsServer) d.Process().Shutdown(t.Context(), errors.New("bye bye")) d.Process().WaitForShutdown() assert.Equal(t, tc.expectedNotifications, appsAPI.Notifications) - assert.Equal(t, tc.expectedWakeUps, appsAPI.WakeUps) assert.Empty(t, mocked.DebugLogger().ErrorMessages()) }) } diff --git a/internal/pkg/service/appsproxy/proxy/testutil/api.go b/internal/pkg/service/appsproxy/proxy/testutil/api.go index 75b8574021..ecf1c7b011 100644 --- a/internal/pkg/service/appsproxy/proxy/testutil/api.go +++ b/internal/pkg/service/appsproxy/proxy/testutil/api.go @@ -21,20 +21,16 @@ import ( type DataAppsAPI struct { *httptest.Server - Apps map[api.AppID]api.AppConfig - Notifications map[string]int - WakeUps map[string]int - WakeUpOverrides map[string]http.HandlerFunc + Apps map[api.AppID]api.AppConfig + Notifications map[string]int } func StartDataAppsAPI(t *testing.T, pm server.PortManager) *DataAppsAPI { t.Helper() service := &DataAppsAPI{ - Apps: make(map[api.AppID]api.AppConfig), - Notifications: make(map[string]int), - WakeUps: make(map[string]int), - WakeUpOverrides: make(map[string]http.HandlerFunc), + Apps: make(map[api.AppID]api.AppConfig), + Notifications: make(map[string]int), } mux := http.NewServeMux() @@ -79,13 +75,6 @@ func StartDataAppsAPI(t *testing.T, pm server.PortManager) *DataAppsAPI { if _, ok := data["lastRequestTimestamp"]; ok { service.Notifications[appID] += 1 } - if _, ok := data["desiredState"]; ok { - if override, ok := service.WakeUpOverrides[appID]; ok { - override(w, req) - return - } - service.WakeUps[appID] += 1 - } }) port := pm.GetFreePort() From 321752cdd4c7e6ec6e13b9e976110cfe38cfc82b Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Mon, 2 Mar 2026 21:13:52 +0100 Subject: [PATCH 02/11] refactor: rename K8s.Namespace to K8s.AppsNamespace Clarifies that the field identifies the namespace where apps (App CRDs) run, not the proxy's own namespace. Env var changes from APPS_PROXY_K8S_NAMESPACE to APPS_PROXY_K8S_APPS_NAMESPACE. --- internal/pkg/service/appsproxy/config/config.go | 4 ++-- internal/pkg/service/appsproxy/dependencies/dependencies.go | 4 ++-- internal/pkg/service/appsproxy/dependencies/mocked.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/pkg/service/appsproxy/config/config.go b/internal/pkg/service/appsproxy/config/config.go index 0527e9585b..24b8cef548 100644 --- a/internal/pkg/service/appsproxy/config/config.go +++ b/internal/pkg/service/appsproxy/config/config.go @@ -44,8 +44,8 @@ type Upstream struct { } type K8s struct { - Namespace string `configKey:"namespace" configUsage:"Kubernetes namespace containing App CRDs." validate:"required"` - Kubeconfig string `configKey:"kubeconfig" configUsage:"Path to kubeconfig file. Uses in-cluster config if empty."` + AppsNamespace string `configKey:"appsNamespace" configUsage:"Kubernetes namespace where apps (App CRDs) run." validate:"required"` + Kubeconfig string `configKey:"kubeconfig" configUsage:"Path to kubeconfig file. Uses in-cluster config if empty."` } func New() Config { diff --git a/internal/pkg/service/appsproxy/dependencies/dependencies.go b/internal/pkg/service/appsproxy/dependencies/dependencies.go index c28ff09227..0040da7898 100644 --- a/internal/pkg/service/appsproxy/dependencies/dependencies.go +++ b/internal/pkg/service/appsproxy/dependencies/dependencies.go @@ -198,7 +198,7 @@ func newServiceScope(ctx context.Context, parentScp parentScopes, cfg config.Con d.appConfigLoader = appconfig.NewLoader(d) d.notifyManager = notify.NewManager(d) - if cfg.K8s.Namespace != "" { + if cfg.K8s.AppsNamespace != "" { var k8sClient dynamic.Interface if provider, ok := parentScp.(k8sClientProvider); ok { k8sClient = provider.K8sDynamicClient() @@ -208,7 +208,7 @@ func newServiceScope(ctx context.Context, parentScp parentScopes, cfg config.Con return nil, err } } - d.appStateWatcher = k8sapp.NewStateWatcher(d, k8sClient, cfg.K8s.Namespace) + d.appStateWatcher = k8sapp.NewStateWatcher(d, k8sClient, cfg.K8s.AppsNamespace) } d.wakeupManager = wakeup.NewManager(d) diff --git a/internal/pkg/service/appsproxy/dependencies/mocked.go b/internal/pkg/service/appsproxy/dependencies/mocked.go index ee7c71fa85..ece7bed7d1 100644 --- a/internal/pkg/service/appsproxy/dependencies/mocked.go +++ b/internal/pkg/service/appsproxy/dependencies/mocked.go @@ -68,8 +68,8 @@ func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config if cfg.SandboxesAPI.Token == "" { cfg.SandboxesAPI.Token = "my-token" } - if cfg.K8s.Namespace == "" { - cfg.K8s.Namespace = "keboola" + if cfg.K8s.AppsNamespace == "" { + cfg.K8s.AppsNamespace = "keboola" } var dnsServer *dnsmock.Server From 47af7e83ff24b8e4f5aada6c74ac95665f776b47 Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Mon, 2 Mar 2026 23:03:21 +0100 Subject: [PATCH 03/11] refactor: make K8s.AppsNamespace required, remove nil guards --- .../appsproxy/dependencies/dependencies.go | 18 ++++++++---------- .../proxy/apphandler/upstream/upstream.go | 15 +++++++-------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/internal/pkg/service/appsproxy/dependencies/dependencies.go b/internal/pkg/service/appsproxy/dependencies/dependencies.go index 0040da7898..c039b79f06 100644 --- a/internal/pkg/service/appsproxy/dependencies/dependencies.go +++ b/internal/pkg/service/appsproxy/dependencies/dependencies.go @@ -198,18 +198,16 @@ func newServiceScope(ctx context.Context, parentScp parentScopes, cfg config.Con d.appConfigLoader = appconfig.NewLoader(d) d.notifyManager = notify.NewManager(d) - if cfg.K8s.AppsNamespace != "" { - var k8sClient dynamic.Interface - if provider, ok := parentScp.(k8sClientProvider); ok { - k8sClient = provider.K8sDynamicClient() - } else { - k8sClient, err = k8sapp.NewDynamicClient(cfg.K8s.Kubeconfig) - if err != nil { - return nil, err - } + var k8sClient dynamic.Interface + if provider, ok := parentScp.(k8sClientProvider); ok { + k8sClient = provider.K8sDynamicClient() + } else { + k8sClient, err = k8sapp.NewDynamicClient(cfg.K8s.Kubeconfig) + if err != nil { + return nil, err } - d.appStateWatcher = k8sapp.NewStateWatcher(d, k8sClient, cfg.K8s.AppsNamespace) } + d.appStateWatcher = k8sapp.NewStateWatcher(d, k8sClient, cfg.K8s.AppsNamespace) d.wakeupManager = wakeup.NewManager(d) d.authProxyManager = authproxy.NewManager(d) diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index 7322f43028..f6bfaa54bd 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -137,16 +137,15 @@ func (m *Manager) NewUpstream(ctx context.Context, app api.AppConfig) (upstream func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request) error { // K8s state pre-check: if we know the app is not running, handle it synchronously // without attempting DNS/upstream. Falls through if state is unknown or Running. - if u.manager.stateWatcher != nil { - if appInfo, ok := u.manager.stateWatcher.GetState(u.app.ID); ok && appInfo.ActualState != k8sapp.AppActualStateRunning { - if !appInfo.AutoRestartEnabled { - u.manager.pageWriter.WriteRestartDisabledPage(rw, req, u.app) - return nil - } - u.wakeup(req.Context(), errors.Errorf("app state is %s", appInfo.ActualState)) - u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + appInfo, ok := u.manager.stateWatcher.GetState(u.app.ID) + if ok && appInfo.ActualState != k8sapp.AppActualStateRunning { + if !appInfo.AutoRestartEnabled { + u.manager.pageWriter.WriteRestartDisabledPage(rw, req, u.app) return nil } + u.wakeup(req.Context(), errors.Errorf("app state is %s", appInfo.ActualState)) + u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + return nil } // Difference between regular and websocket request From e980dd159d867ff599bc77abcde1a3693059b4db Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Mon, 2 Mar 2026 21:28:33 +0100 Subject: [PATCH 04/11] debug: add app state check logs in apps-proxy request handling --- .../pkg/service/appsproxy/dataapps/k8sapp/watcher.go | 1 + .../appsproxy/proxy/apphandler/upstream/upstream.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go index 61bd205139..43836c1681 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go @@ -187,6 +187,7 @@ func (w *StateWatcher) handleUpsert(ctx context.Context, obj any) { appID := api.AppID(appObj.Spec.AppID) w.byName.Store(k8sName, appID) w.byAppID.Store(appID, entry{k8sName: k8sName, state: appObj.Status.CurrentState, autoRestartEnabled: autoRestartEnabled}) + w.logger.Debugf(ctx, "App CRD %q (appID=%s) state updated: actualState=%q autoRestartEnabled=%v", k8sName, appID, appObj.Status.CurrentState, autoRestartEnabled) } func (w *StateWatcher) handleDelete(ctx context.Context, obj any) { diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index f6bfaa54bd..86d8853c18 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -135,15 +135,24 @@ func (m *Manager) NewUpstream(ctx context.Context, app api.AppConfig) (upstream } func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request) error { + ctx := req.Context() + // K8s state pre-check: if we know the app is not running, handle it synchronously // without attempting DNS/upstream. Falls through if state is unknown or Running. appInfo, ok := u.manager.stateWatcher.GetState(u.app.ID) + if ok { + u.manager.logger.Debugf(ctx, "app %q state check: actualState=%q autoRestartEnabled=%v", u.app.ID, appInfo.ActualState, appInfo.AutoRestartEnabled) + } else { + u.manager.logger.Debugf(ctx, "app %q state check: not in cache, forwarding to upstream", u.app.ID) + } if ok && appInfo.ActualState != k8sapp.AppActualStateRunning { if !appInfo.AutoRestartEnabled { + u.manager.logger.Debugf(ctx, "app %q is not running and restart is disabled, serving restart-disabled page", u.app.ID) u.manager.pageWriter.WriteRestartDisabledPage(rw, req, u.app) return nil } - u.wakeup(req.Context(), errors.Errorf("app state is %s", appInfo.ActualState)) + u.manager.logger.Debugf(ctx, "app %q is not running, triggering wakeup and serving spinner page", u.app.ID) + u.wakeup(ctx, errors.Errorf("app state is %s", appInfo.ActualState)) u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) return nil } From ce117da665dcba335acd4c49b6119ecb8d02ee7c Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Mon, 2 Mar 2026 21:34:40 +0100 Subject: [PATCH 05/11] fix: show spinner page when app state is Starting, not restart-disabled page --- .../service/appsproxy/proxy/apphandler/upstream/upstream.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index 86d8853c18..f6e0817f9a 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -146,6 +146,11 @@ func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request u.manager.logger.Debugf(ctx, "app %q state check: not in cache, forwarding to upstream", u.app.ID) } if ok && appInfo.ActualState != k8sapp.AppActualStateRunning { + if appInfo.ActualState == k8sapp.AppActualStateStarting { + u.manager.logger.Debugf(ctx, "app %q is starting, serving spinner page", u.app.ID) + u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + return nil + } if !appInfo.AutoRestartEnabled { u.manager.logger.Debugf(ctx, "app %q is not running and restart is disabled, serving restart-disabled page", u.app.ID) u.manager.pageWriter.WriteRestartDisabledPage(rw, req, u.app) From 475840a0932aa1e243d46f827a41086c4744ea85 Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Mon, 2 Mar 2026 22:47:07 +0100 Subject: [PATCH 06/11] refactor: derive upstream URL from App CR appsProxyServiceRef.name Previously the upstream URL was provided directly as a string by the backend (operator) in .status.appsProxyServiceRef. The operator changed the field to an object {name: ""}, so the proxy now constructs the upstream URL as http://..svc.cluster.local:8888. - Parse appsProxyServiceRef as {name} object in StateWatcher; construct upstream URL from service name, namespace, and static port 8888 - Add StateWatcher.SetServiceURLBuilder to let tests redirect traffic to a local test server instead of a real K8s service - Store target URL in AppUpstream at creation time (immutable field); replace Rewrite-based ReverseProxy with NewSingleHostReverseProxy - Add Manager.CurrentServiceRef to expose the cached URL string for change detection - Recreate the app handler in HandlerFor when appsProxyServiceRef changes (wrapper.serviceRef != currentRef), same mechanism used for API config changes - Register all test apps in K8s watcher before proxy requests in tests --- .../appsproxy/dataapps/k8sapp/types.go | 16 +++- .../appsproxy/dataapps/k8sapp/watcher.go | 44 +++++++++-- .../appsproxy/dataapps/k8sapp/watcher_test.go | 58 ++++++++++++++ .../appsproxy/proxy/apphandler/manager.go | 13 ++-- .../proxy/apphandler/upstream/upstream.go | 30 ++++++-- .../pkg/service/appsproxy/proxy/proxy_test.go | 76 ++++++++++++++----- .../service/appsproxy/proxy/server_test.go | 6 +- 7 files changed, 199 insertions(+), 44 deletions(-) diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go index c78080ef6d..d7112736b8 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go @@ -1,7 +1,11 @@ // Package k8sapp provides watching and patching of App CRDs in Kubernetes. package k8sapp -import "k8s.io/apimachinery/pkg/runtime/schema" +import ( + "net/url" + + "k8s.io/apimachinery/pkg/runtime/schema" +) const ( Group = "apps.keboola.com" @@ -37,8 +41,16 @@ type appSpec struct { type AppInfo struct { ActualState AppActualState AutoRestartEnabled bool + // UpstreamTarget is the pre-parsed URL from .status.appsProxyServiceRef. + // Nil when the field is absent or unparseable. + UpstreamTarget *url.URL } type appStatus struct { - CurrentState AppActualState `json:"currentState"` + CurrentState AppActualState `json:"currentState"` + AppsProxyServiceRef appsProxyServiceRef `json:"appsProxyServiceRef,omitempty"` +} + +type appsProxyServiceRef struct { + Name string `json:"name,omitempty"` } diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go index 43836c1681..eb582cc9b4 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go @@ -3,6 +3,8 @@ package k8sapp import ( "context" "encoding/json" + "fmt" + "net/url" "sync" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,13 +27,15 @@ type entry struct { k8sName string state AppActualState autoRestartEnabled bool + upstreamTarget *url.URL // pre-parsed; nil when appsProxyServiceRef absent/invalid } // StateWatcher watches App CRDs in Kubernetes and provides a local cache of app states. type StateWatcher struct { - client dynamic.Interface - namespace string - logger log.Logger + client dynamic.Interface + namespace string + logger log.Logger + serviceURLBuilder func(name string) string // byName: K8s object name → AppID byName sync.Map // byAppID: AppID → entry{k8sName, state} @@ -66,6 +70,9 @@ func NewStateWatcher(d dependencies, client dynamic.Interface, namespace string) client: client, namespace: namespace, logger: d.Logger().WithComponent("k8sapp.watcher"), + serviceURLBuilder: func(name string) string { + return fmt.Sprintf("http://%s.%s.svc.cluster.local:8888", name, namespace) + }, } ctx, cancel := context.WithCancel(context.Background()) @@ -124,7 +131,17 @@ func (w *StateWatcher) GetState(appID api.AppID) (AppInfo, bool) { return AppInfo{}, false } e := v.(entry) - return AppInfo{ActualState: e.state, AutoRestartEnabled: e.autoRestartEnabled}, true + return AppInfo{ + ActualState: e.state, + AutoRestartEnabled: e.autoRestartEnabled, + UpstreamTarget: e.upstreamTarget, + }, true +} + +// SetServiceURLBuilder overrides the function used to construct the upstream URL from a service name. +// It is intended for use in tests only, to point the proxy at a local test server. +func (w *StateWatcher) SetServiceURLBuilder(f func(name string) string) { + w.serviceURLBuilder = f } // SetDesiredRunning patches .spec.state = "Running" on the App CRD for the given appID. @@ -184,10 +201,25 @@ func (w *StateWatcher) handleUpsert(ctx context.Context, obj any) { autoRestartEnabled = *appObj.Spec.AutoRestartEnabled } + var upstreamTarget *url.URL + if name := appObj.Status.AppsProxyServiceRef.Name; name != "" { + rawURL := w.serviceURLBuilder(name) + if t, err := url.Parse(rawURL); err == nil { + upstreamTarget = t + } else { + w.logger.Warnf(ctx, "App CRD %q (appID=%s) invalid upstream URL %q for appsProxyServiceRef %q: %s", k8sName, appObj.Spec.AppID, rawURL, name, err) + } + } + appID := api.AppID(appObj.Spec.AppID) w.byName.Store(k8sName, appID) - w.byAppID.Store(appID, entry{k8sName: k8sName, state: appObj.Status.CurrentState, autoRestartEnabled: autoRestartEnabled}) - w.logger.Debugf(ctx, "App CRD %q (appID=%s) state updated: actualState=%q autoRestartEnabled=%v", k8sName, appID, appObj.Status.CurrentState, autoRestartEnabled) + w.byAppID.Store(appID, entry{ + k8sName: k8sName, + state: appObj.Status.CurrentState, + autoRestartEnabled: autoRestartEnabled, + upstreamTarget: upstreamTarget, + }) + w.logger.Debugf(ctx, "App CRD %q (appID=%s) state updated: actualState=%q autoRestartEnabled=%v upstreamTarget=%v", k8sName, appID, appObj.Status.CurrentState, autoRestartEnabled, upstreamTarget != nil) } func (w *StateWatcher) handleDelete(ctx context.Context, obj any) { diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go index 6d40a147b7..0c16d8d1d0 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go @@ -71,6 +71,13 @@ func newAppObject(k8sName, appID string, state k8sapp.AppActualState) *unstructu } } +// newAppObjectWithServiceRef creates an unstructured App CRD object with appsProxyServiceRef set. +func newAppObjectWithServiceRef(k8sName, appID string, state k8sapp.AppActualState, serviceName string) *unstructured.Unstructured { + obj := newAppObject(k8sName, appID, state) + obj.Object["status"].(map[string]any)["appsProxyServiceRef"] = map[string]any{"name": serviceName} + return obj +} + func TestStateWatcher_GetState_UnknownWhenEmpty(t *testing.T) { t.Parallel() @@ -153,3 +160,54 @@ func TestStateWatcher_SetDesiredRunning_NoOpWhenUnknown(t *testing.T) { assert.NotEqual(t, "patch", a.GetVerb(), "unexpected PATCH for unknown app") } } + +func TestStateWatcher_GetState_UpstreamTarget(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + appObj := newAppObjectWithServiceRef("my-app-k8s", "app-123", k8sapp.AppActualStateRunning, "svc-name") + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + var info k8sapp.AppInfo + assert.Eventually(t, func() bool { + var ok bool + info, ok = watcher.GetState(api.AppID("app-123")) + return ok && info.UpstreamTarget != nil + }, 5*time.Second, 50*time.Millisecond) + + require.NotNil(t, info.UpstreamTarget) + assert.Equal(t, "http", info.UpstreamTarget.Scheme) + assert.Equal(t, "svc-name.keboola.svc.cluster.local:8888", info.UpstreamTarget.Host) +} + +func TestStateWatcher_GetState_UpstreamTarget_AbsentWhenMissing(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + // App CRD without appsProxyServiceRef. + appObj := newAppObject("my-app-k8s", "app-123", k8sapp.AppActualStateRunning) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + assert.Eventually(t, func() bool { + _, ok := watcher.GetState(api.AppID("app-123")) + return ok + }, 5*time.Second, 50*time.Millisecond) + + info, ok := watcher.GetState(api.AppID("app-123")) + require.True(t, ok) + assert.Nil(t, info.UpstreamTarget) +} diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/manager.go b/internal/pkg/service/appsproxy/proxy/apphandler/manager.go index c72b5844ae..3c07033710 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/manager.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/manager.go @@ -29,9 +29,10 @@ type Manager struct { } type appHandlerWrapper struct { - lock *sync.Mutex - handler http.Handler - cancel context.CancelCauseFunc + lock *sync.Mutex + handler http.Handler + cancel context.CancelCauseFunc + serviceRef string // appsProxyServiceRef in effect when this handler was created } type dependencies interface { @@ -70,12 +71,14 @@ func (m *Manager) HandlerFor(ctx context.Context, result appconfig.AppConfigResu return m.newErrorHandler(ctx, api.AppConfig{ID: result.AppID}, result.Err) } - // Create a new handler, if needed - if wrapper.handler == nil || result.Modified { + // Create a new handler, if needed (config changed or appsProxyServiceRef changed) + currentRef := m.upstreamManager.CurrentServiceRef(result.AppID) + if wrapper.handler == nil || result.Modified || wrapper.serviceRef != currentRef { if wrapper.cancel != nil { wrapper.cancel(errors.New("configuration changed")) } wrapper.handler, wrapper.cancel = m.newHandler(ctx, result.AppConfig) + wrapper.serviceRef = currentRef } return wrapper.handler diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index f6e0817f9a..26c15766f8 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -24,7 +24,6 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/chain" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/pagewriter" "github.com/keboola/keboola-as-code/internal/pkg/service/common/ctxattr" - svcErrors "github.com/keboola/keboola-as-code/internal/pkg/service/common/errors" "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" @@ -53,7 +52,7 @@ type Manager struct { type AppUpstream struct { manager *Manager app api.AppConfig - target *url.URL + target *url.URL // parsed from appsProxyServiceRef at creation; nil when absent handler *chain.Chain wsHandler *chain.Chain cancelWs context.CancelCauseFunc @@ -99,16 +98,24 @@ func (m *Manager) Shutdown(ctx context.Context) { m.wg.Wait() } +// CurrentServiceRef returns the upstream URL string for appID from the K8s cache. +// Returns "" when the app is not cached or the field is absent/invalid. +func (m *Manager) CurrentServiceRef(appID api.AppID) string { + info, ok := m.stateWatcher.GetState(appID) + if !ok || info.UpstreamTarget == nil { + return "" + } + return info.UpstreamTarget.String() +} + func (m *Manager) NewUpstream(ctx context.Context, app api.AppConfig) (upstream *AppUpstream, err error) { _, span := m.telemetry.Tracer().Start(ctx, "keboola.go.apps-proxy.upstream.NewUpstream") defer span.End(&err) - // Parse target - target, err := url.Parse(app.UpstreamAppURL) - if err != nil { - return nil, svcErrors.NewServiceUnavailableError(errors.PrefixErrorf(err, - `unable to parse upstream url for app "%s"`, app.IdAndName(), - )) + // Resolve target URL at creation time; immutable after this point. + var target *url.URL + if info, ok := m.stateWatcher.GetState(app.ID); ok { + target = info.UpstreamTarget // pre-parsed by watcher on CRD event; may be nil } // Create reverse proxy @@ -162,6 +169,13 @@ func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request return nil } + // Target set at creation time; nil means appsProxyServiceRef was absent. + if u.target == nil { + u.manager.logger.Debugf(ctx, "app %q has no appsProxyServiceRef, serving spinner page", u.app.ID) + u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + return nil + } + // Difference between regular and websocket request if strings.EqualFold(req.Header.Get("Connection"), "upgrade") && req.Header.Get("Upgrade") == "websocket" { return u.wsHandler.ServeHTTPOrError(rw, req) diff --git a/internal/pkg/service/appsproxy/proxy/proxy_test.go b/internal/pkg/service/appsproxy/proxy/proxy_test.go index 269114286e..638ed0bad3 100644 --- a/internal/pkg/service/appsproxy/proxy/proxy_test.go +++ b/internal/pkg/service/appsproxy/proxy/proxy_test.go @@ -38,6 +38,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8stypes "k8s.io/apimachinery/pkg/types" k8sfake "k8s.io/client-go/dynamic/fake" "github.com/keboola/keboola-as-code/internal/pkg/log" @@ -2186,26 +2187,11 @@ func TestAppProxyRouter(t *testing.T) { { name: "restart-disabled", setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { - autoRestartEnabled := false - appObj := &unstructured.Unstructured{ - Object: map[string]any{ - "apiVersion": k8sapp.Group + "/" + k8sapp.Version, - "kind": "App", - "metadata": map[string]any{ - "name": "app-123", - "namespace": "keboola", - }, - "spec": map[string]any{ - "appId": "123", - "autoRestartEnabled": autoRestartEnabled, - }, - "status": map[string]any{ - "currentState": string(k8sapp.AppActualStateStopped), - }, - }, - } - _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Create( - t.Context(), appObj, metav1.CreateOptions{}, + // The default setup already created app "123" as Running. + // Patch it to Stopped + autoRestartEnabled=false. + patch := []byte(`{"spec":{"autoRestartEnabled":false},"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, ) require.NoError(t, err) @@ -2398,7 +2384,8 @@ func TestAppProxyRouter(t *testing.T) { // Register data apps appURL := testutil.AddAppDNSRecord(t, appServer, dnsServer) - appsAPI.Register(testDataApps(appURL, providers)) + apps := testDataApps(appURL, providers) + appsAPI.Register(apps) // Create proxy handler handler := createProxyHandler(ctx, d) @@ -2426,6 +2413,8 @@ func TestAppProxyRouter(t *testing.T) { client := createHTTPClient(t, proxyURL) + // Default K8s setup: register all test apps as Running with appsProxyServiceRef. + registerDefaultK8sApps(t, apps, mocked.TestFakeK8sClient(), d.AppStateWatcher(), appURL.String()) if tc.setupK8s != nil { tc.setupK8s(t, mocked.TestFakeK8sClient(), d.AppStateWatcher()) } @@ -2830,3 +2819,48 @@ func extractMetaRefreshTag(t *testing.T, body []byte) string { func htmlLinkTo(url string) string { return fmt.Sprintf(` Date: Tue, 3 Mar 2026 13:37:55 +0100 Subject: [PATCH 07/11] refactor: remove custom DNS resolution from apps-proxy transport App state is now read exclusively from the App CR, so the DNS-based wakeup fallback and the custom non-recursive DNS client are no longer needed. Replace the custom DNS transport with standard Go dialer and update tests to use App CR state (Stopped/Running) instead of DNS record manipulation to simulate app unavailability. --- .../pkg/service/appsproxy/config/config.go | 1 - .../appsproxy/dependencies/dependencies.go | 4 +- .../service/appsproxy/dependencies/mocked.go | 19 +- .../proxy/apphandler/upstream/upstream.go | 5 - .../pkg/service/appsproxy/proxy/proxy_test.go | 203 +++++++++++------- .../service/appsproxy/proxy/server_test.go | 2 +- .../service/appsproxy/proxy/testutil/dns.go | 36 +--- .../appsproxy/proxy/transport/dialer.go | 2 +- .../appsproxy/proxy/transport/dns/client.go | 108 ---------- .../proxy/transport/dns/dnsmock/dnsmock.go | 186 ---------------- .../transport/dns/dnsmock/dnsmock_test.go | 101 --------- .../appsproxy/proxy/transport/transport.go | 92 +------- 12 files changed, 136 insertions(+), 623 deletions(-) delete mode 100644 internal/pkg/service/appsproxy/proxy/transport/dns/client.go delete mode 100644 internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock.go delete mode 100644 internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock_test.go diff --git a/internal/pkg/service/appsproxy/config/config.go b/internal/pkg/service/appsproxy/config/config.go index 24b8cef548..1d9dc41b03 100644 --- a/internal/pkg/service/appsproxy/config/config.go +++ b/internal/pkg/service/appsproxy/config/config.go @@ -19,7 +19,6 @@ type Config struct { PProf pprof.Config `configKey:"pprof"` Datadog datadog.Config `configKey:"datadog"` Metrics prometheus.Config `configKey:"metrics"` - DNSServer string `configKey:"dnsServer" configUsage:"DNS server for proxy. If empty, the /etc/resolv.conf is used."` API API `configKey:"api"` CookieSecretSalt string `configKey:"cookieSecretSalt" configUsage:"Cookie secret needed by OAuth 2 Proxy." validate:"required" sensitive:"true"` Upstream Upstream `configKey:"-" configUsage:"Configuration options for upstream"` diff --git a/internal/pkg/service/appsproxy/dependencies/dependencies.go b/internal/pkg/service/appsproxy/dependencies/dependencies.go index c039b79f06..b3f6be28e3 100644 --- a/internal/pkg/service/appsproxy/dependencies/dependencies.go +++ b/internal/pkg/service/appsproxy/dependencies/dependencies.go @@ -35,7 +35,6 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/upstream" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/pagewriter" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/common/httpclient" "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" @@ -88,7 +87,6 @@ type ServiceScope interface { type Mocked interface { dependencies.Mocked TestConfig() config.Config - TestDNSServer() *dnsmock.Server TestFakeK8sClient() *k8sfake.FakeDynamicClient } @@ -184,7 +182,7 @@ func newServiceScope(ctx context.Context, parentScp parentScopes, cfg config.Con d.parentScopes = parentScp d.config = cfg - d.upstreamTransport, err = transport.New(d, cfg.DNSServer) + d.upstreamTransport, err = transport.New(d) if err != nil { return nil, err } diff --git a/internal/pkg/service/appsproxy/dependencies/mocked.go b/internal/pkg/service/appsproxy/dependencies/mocked.go index ece7bed7d1..f66eea85d6 100644 --- a/internal/pkg/service/appsproxy/dependencies/mocked.go +++ b/internal/pkg/service/appsproxy/dependencies/mocked.go @@ -13,7 +13,6 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" ) @@ -22,7 +21,6 @@ import ( type mocked struct { dependencies.Mocked config config.Config - dnsServer *dnsmock.Server fakeK8sClient *k8sfake.FakeDynamicClient } @@ -30,10 +28,6 @@ func (v *mocked) TestConfig() config.Config { return v.config } -func (v *mocked) TestDNSServer() *dnsmock.Server { - return v.dnsServer -} - // TestFakeK8sClient returns the fake Kubernetes dynamic client used by this mock. // Tests can use it to pre-populate App CRD objects and inspect PATCH actions. func (v *mocked) TestFakeK8sClient() *k8sfake.FakeDynamicClient { @@ -72,17 +66,6 @@ func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config cfg.K8s.AppsNamespace = "keboola" } - var dnsServer *dnsmock.Server - if cfg.DNSServer == "" { - dnsServer = dnsmock.New(commonMock.MockedDNSPort()) - require.NoError(tb, dnsServer.Start()) - tb.Cleanup(func() { - _ = dnsServer.Shutdown() - }) - - cfg.DNSServer = dnsServer.Addr() - } - // Create fake K8s dynamic client. The App list kind is registered so the informer can list CRDs. scheme := runtime.NewScheme() fakeClient := k8sfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ @@ -92,7 +75,7 @@ func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config // Validate config require.NoError(tb, configmap.ValidateAndNormalize(&cfg)) - mock := &mocked{Mocked: commonMock, config: cfg, dnsServer: dnsServer, fakeK8sClient: fakeClient} + mock := &mocked{Mocked: commonMock, config: cfg, fakeK8sClient: fakeClient} scope, err := newServiceScope(ctx, mock, cfg) require.NoError(tb, err) diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index 26c15766f8..4f525d1848 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -238,11 +238,6 @@ func (u *AppUpstream) trace() chain.Middleware { GotConn: func(connInfo httptrace.GotConnInfo) { u.notify(ctx) }, - DNSDone: func(info httptrace.DNSDoneInfo) { - if info.Err != nil { - u.wakeup(ctx, info.Err) - } - }, }) return next.ServeHTTPOrError(w, req.WithContext(reqCtx)) diff --git a/internal/pkg/service/appsproxy/proxy/proxy_test.go b/internal/pkg/service/appsproxy/proxy/proxy_test.go index 638ed0bad3..273d296d2a 100644 --- a/internal/pkg/service/appsproxy/proxy/proxy_test.go +++ b/internal/pkg/service/appsproxy/proxy/proxy_test.go @@ -23,7 +23,6 @@ import ( "github.com/coder/websocket" "github.com/coder/websocket/wsjson" "github.com/keboola/go-utils/pkg/wildcards" - "github.com/miekg/dns" "github.com/oauth2-proxy/mockoidc" "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/logger" "github.com/stretchr/testify/assert" @@ -51,7 +50,6 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/logging" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/pagewriter" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/testutil" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" @@ -61,7 +59,7 @@ import ( type testCase struct { name string setupK8s func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) - run func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) + run func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) expectedNotifications map[string]int expectedSpans tracetest.SpanStubs } @@ -72,7 +70,7 @@ func TestAppProxyRouter(t *testing.T) { testCases := []testCase{ { name: "health-check", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to health-check endpoint request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://hub.keboola.local/health-check", nil) require.NoError(t, err) @@ -87,7 +85,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "missing-app-id", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request without app id request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://hub.keboola.local/", nil) require.NoError(t, err) @@ -102,7 +100,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "unknown-app-id", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to unknown app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://unknown.hub.keboola.local/health-check", nil) require.NoError(t, err) @@ -117,7 +115,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "broken-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to broken app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://broken.hub.keboola.local/", nil) require.NoError(t, err) @@ -221,7 +219,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "no-rule-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with no path rules request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://norule.hub.keboola.local/", nil) require.NoError(t, err) @@ -237,7 +235,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "wrong-rule-type-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid1.hub.keboola.local/", nil) require.NoError(t, err) @@ -253,7 +251,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "missing-referenced-provider", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid2.hub.keboola.local/", nil) require.NoError(t, err) @@ -269,7 +267,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "empty-allowed-roles-array-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid3.hub.keboola.local/", nil) require.NoError(t, err) @@ -285,7 +283,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "unknown-provider-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with unknown provider request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid4.hub.keboola.local/", nil) require.NoError(t, err) @@ -301,7 +299,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "not-empty-providers-auth-required-false", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid5.hub.keboola.local/", nil) require.NoError(t, err) @@ -317,7 +315,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "redirect-to-canonical-host", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Redirect to the canonical URL (match cookies domain) request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://foo-bar-123.hub.keboola.local/some/data/app/url?foo=bar", nil) require.NoError(t, err) @@ -331,7 +329,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "redirect-to-host-lowercase", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://foo-12345.hub.keboola.local/some/data/app/url?foo=bar", nil) require.NoError(t, err) response, err := client.Do(request) @@ -344,7 +342,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-app-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() // Request to public app @@ -361,7 +359,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-app-sub-url", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/some/data/app/url?foo=bar", nil) request.Header.Set("User-Agent", "Internet Exploder") @@ -391,7 +389,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-verified-email", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(true), @@ -477,7 +475,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-unauthorized", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to private app (unauthorized) request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) @@ -511,7 +509,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-missing-csrf-token", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -581,7 +579,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-group-mismatch", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "manager@keboola.com", Groups: []string{"manager"}, @@ -635,7 +633,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-unverified-email", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(false), @@ -687,7 +685,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-oidc-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to private app (unauthorized) request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) @@ -738,7 +736,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() m[0].QueueUser(&mockoidc.MockUser{ @@ -829,7 +827,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "multi-app-basic-flow", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin", "manager"}, @@ -934,7 +932,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "multi-app-redirect-to-selection-page", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -965,7 +963,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "multi-app-unverified-email", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(false), @@ -1033,7 +1031,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "multi-app-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() m[1].QueueUser(&mockoidc.MockUser{ @@ -1110,7 +1108,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "multi-app-broken-provider", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() // Provider selection @@ -1137,7 +1135,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-app-websocket", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), time.Minute) defer cancel() @@ -1164,7 +1162,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-websocket-unauthorized", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), time.Minute) defer cancel() @@ -1182,7 +1180,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-websocket", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(true), @@ -1254,7 +1252,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "websocket-connection-check", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() @@ -1284,7 +1282,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "multi-app-websocket", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1376,7 +1374,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "prefix-app-no-auth", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public part of the app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://prefix.hub.keboola.local/public", nil) require.NoError(t, err) @@ -1411,7 +1409,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "prefix-app-api-auth", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1473,7 +1471,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "prefix-app-web-auth", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1554,7 +1552,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "shared-provider", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1641,7 +1639,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "configuration-change", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) @@ -1687,7 +1685,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "concurrency-test", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() @@ -1723,17 +1721,35 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-app-wakeup", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - - // Request to public app - fails because the app doesn't have a DNS record + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + // Request to public app - fails because the app is Stopped request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) response, err := client.Do(request) require.NoError(t, err) require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - dnsServer.AddARecord(dns.Fqdn("app.local"), net.ParseIP("127.0.0.1")) + // Patch app back to Running + patch := []byte(`{"status":{"currentState":"Running"}}`) + _, err = fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && info.ActualState == k8sapp.AppActualStateRunning + }, 5*time.Second, 50*time.Millisecond) // Request to public app request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) @@ -1751,10 +1767,19 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-app-wakeup-only", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - - // Request to public app - fails because the app doesn't have a DNS record + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + // Request to public app - fails because the app is Stopped, triggers wakeup request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) response, err := client.Do(request) @@ -1771,7 +1796,7 @@ func TestAppProxyRouter(t *testing.T) { { name: "private-one-provider-selector", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request provider selector page - no auth provider request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/_proxy/selection", nil) require.NoError(t, err) @@ -1786,9 +1811,18 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-wakeup", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-oidc", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("oidc")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1835,14 +1869,23 @@ func TestAppProxyRouter(t *testing.T) { require.NoError(t, err) require.Equal(t, http.StatusFound, response.StatusCode) - // Request to private app (authorized but missing dns, triggers wakeup) + // Request to private app (authorized but app is Stopped, triggers wakeup) request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) response, err = client.Do(request) require.NoError(t, err) require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - dnsServer.AddARecord(dns.Fqdn("app.local"), net.ParseIP("127.0.0.1")) + // Patch app back to Running + patch := []byte(`{"status":{"currentState":"Running"}}`) + _, err = fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-oidc", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("oidc")) + return ok && info.ActualState == k8sapp.AppActualStateRunning + }, 5*time.Second, 50*time.Millisecond) // Request to private app (authorized) request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) @@ -1857,9 +1900,18 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-wakeup-only", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-oidc", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("oidc")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1906,7 +1958,7 @@ func TestAppProxyRouter(t *testing.T) { require.NoError(t, err) require.Equal(t, http.StatusFound, response.StatusCode) - // Request to private app (authorized but missing dns, triggers wakeup) + // Request to private app (authorized but app is Stopped, triggers wakeup) request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) response, err = client.Do(request) @@ -1922,9 +1974,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "private-app-no-wakeup", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1977,7 +2027,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-basic-auth-wrong-login-no-password", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/", nil) require.NoError(t, err) @@ -2003,7 +2053,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-basic-auth-wrong-login", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/", nil) require.NoError(t, err) @@ -2029,7 +2079,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-basic-auth-correct-app-url", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/app/url", nil) require.NoError(t, err) @@ -2076,7 +2126,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-basic-auth-correct-login", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/_proxy/form", nil) require.NoError(t, err) @@ -2126,7 +2176,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-basic-auth-wrong-cookie", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Access with cookie request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://basic-auth.hub.keboola.local/", nil) request.AddCookie(&http.Cookie{Name: "proxyBasicAuth", Value: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015a"}) @@ -2142,7 +2192,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-basic-auth-cookie", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Access with cookie request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://basic-auth.hub.keboola.local/", nil) request.AddCookie(&http.Cookie{Name: "proxyBasicAuth", Value: "$2a$10$65mF6LI2F0Nm9PkQk8DlJu.C5jD.fseeXWn9CCGmDxLPomikYWtte"}) @@ -2161,7 +2211,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "public-basic-auth-sign-out", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Access with cookie request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://basic-auth.hub.keboola.local/_proxy/sign_out", nil) request.AddCookie(&http.Cookie{Name: "proxyBasicAuth", Value: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"}) @@ -2200,7 +2250,7 @@ func TestAppProxyRouter(t *testing.T) { return ok && !info.AutoRestartEnabled }, 5*time.Second, 50*time.Millisecond) }, - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Use canonical slug-based URL to bypass the slug redirect. request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) @@ -2218,7 +2268,7 @@ func TestAppProxyRouter(t *testing.T) { publicAppTestCaseFactory := func(method string) testCase { return testCase{ name: "public-app-" + method, - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public app request, err := http.NewRequestWithContext(t.Context(), method, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) @@ -2247,7 +2297,7 @@ func TestAppProxyRouter(t *testing.T) { privateAppTestCaseFactory := func(method string) testCase { return testCase{ name: "private-app-oidc-" + method, - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -2377,13 +2427,11 @@ func TestAppProxyRouter(t *testing.T) { mocked.TestTelemetry().AssertSpans(t, tc.expectedSpans, opts...) } - dnsServer := mocked.TestDNSServer() - // Create test OIDC providers providers := testAuthProviders(t, pm) // Register data apps - appURL := testutil.AddAppDNSRecord(t, appServer, dnsServer) + appURL := testutil.AppServerURL(t, appServer) apps := testDataApps(appURL, providers) appsAPI.Register(apps) @@ -2419,7 +2467,7 @@ func TestAppProxyRouter(t *testing.T) { tc.setupK8s(t, mocked.TestFakeK8sClient(), d.AppStateWatcher()) } - tc.run(t, client, providers, appServer, appsAPI, dnsServer) + tc.run(t, client, providers, appServer, appsAPI, mocked.TestFakeK8sClient(), d.AppStateWatcher()) d.Process().Shutdown(t.Context(), errors.New("bye bye")) d.Process().WaitForShutdown() @@ -2760,8 +2808,7 @@ func createDependencies(t *testing.T, ctx context.Context, sandboxesAPIURL strin cfg.CookieSecretSalt = string(secret) cfg.CsrfTokenSalt = string(csrfSecret) cfg.SandboxesAPI.URL = sandboxesAPIURL - port := pm.GetFreePort() - return proxyDependencies.NewMockedServiceScope(t, ctx, cfg, dependencies.WithRealHTTPClient(), dependencies.WithMockedDNSPort(port)) + return proxyDependencies.NewMockedServiceScope(t, ctx, cfg, dependencies.WithRealHTTPClient()) } func createProxyHandler(ctx context.Context, d proxyDependencies.ServiceScope) http.Handler { diff --git a/internal/pkg/service/appsproxy/proxy/server_test.go b/internal/pkg/service/appsproxy/proxy/server_test.go index 9e7eab16c3..70759f2728 100644 --- a/internal/pkg/service/appsproxy/proxy/server_test.go +++ b/internal/pkg/service/appsproxy/proxy/server_test.go @@ -60,7 +60,7 @@ func TestAppProxyHandler(t *testing.T) { d, mocked := proxyDependencies.NewMockedServiceScope(t, ctx, cfg, dependencies.WithRealHTTPClient()) // Register apps - appURL := testutil.AddAppDNSRecord(t, appServer, mocked.TestDNSServer()) + appURL := testutil.AppServerURL(t, appServer) apps := []api.AppConfig{ { ID: "123", diff --git a/internal/pkg/service/appsproxy/proxy/testutil/dns.go b/internal/pkg/service/appsproxy/proxy/testutil/dns.go index 0f7df418e3..78e6ed3e96 100644 --- a/internal/pkg/service/appsproxy/proxy/testutil/dns.go +++ b/internal/pkg/service/appsproxy/proxy/testutil/dns.go @@ -1,45 +1,17 @@ package testutil import ( - "net" "net/url" "testing" - "github.com/miekg/dns" "github.com/stretchr/testify/require" - - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" ) -func StartDNSServer(t *testing.T, port int) *dnsmock.Server { +// AppServerURL returns the URL of the app server for use as the upstream target. +func AppServerURL(t *testing.T, appServer *AppServer) *url.URL { t.Helper() - server := dnsmock.New(port) - err := server.Start() + u, err := url.Parse(appServer.URL) require.NoError(t, err) - - return server -} - -func AddAppDNSRecord(t *testing.T, appServer *AppServer, dnsServer *dnsmock.Server) (appURL *url.URL) { - t.Helper() - - tsURL, err := url.Parse(appServer.URL) - require.NoError(t, err) - - ip, _, err := net.SplitHostPort(tsURL.Host) - require.NoError(t, err) - - appHost := "app.local" - var derr *dnsmock.DNSRecordError - err = dnsServer.AddAOrAAAARecord(dns.Fqdn(appHost), net.ParseIP(ip)) - if err != nil && errors.As(err, &derr) { - return nil - } - - return &url.URL{ - Scheme: tsURL.Scheme, - Host: net.JoinHostPort(appHost, tsURL.Port()), - } + return u } diff --git a/internal/pkg/service/appsproxy/proxy/transport/dialer.go b/internal/pkg/service/appsproxy/proxy/transport/dialer.go index 937c1e5b55..c6464d9b9a 100644 --- a/internal/pkg/service/appsproxy/proxy/transport/dialer.go +++ b/internal/pkg/service/appsproxy/proxy/transport/dialer.go @@ -11,7 +11,7 @@ const DialTimeout = 2 * time.Second // KeepAlive specifies the default interval between keep-alive probes. const KeepAlive = 15 * time.Second -// newDialer creates dialer for DNS resolving and the HTTP transport. +// newDialer creates dialer for the HTTP transport. func newDialer() *net.Dialer { return &net.Dialer{ Timeout: DialTimeout, diff --git a/internal/pkg/service/appsproxy/proxy/transport/dns/client.go b/internal/pkg/service/appsproxy/proxy/transport/dns/client.go deleted file mode 100644 index 092a46c6bb..0000000000 --- a/internal/pkg/service/appsproxy/proxy/transport/dns/client.go +++ /dev/null @@ -1,108 +0,0 @@ -package dns - -import ( - "context" - "fmt" - "math/rand" - "net" - "strings" - "time" - - "github.com/miekg/dns" - - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" -) - -const DialTimeout = 2 * time.Second - -const ReadTimeout = 2 * time.Second - -const WriteTimeout = 2 * time.Second - -type Client struct { - client *dns.Client - dnsServer string -} - -func NewClientFromEtc(dialer *net.Dialer) (*Client, error) { - // Parse DNS config - dnsCfgFile := "/etc/resolv.conf" - dnsCfg, err := dns.ClientConfigFromFile(dnsCfgFile) - if err != nil { - return nil, err - } - if len(dnsCfg.Servers) == 0 { - return nil, errors.Errorf(`no DNS server found in "%s"`, dnsCfgFile) - } - - // Get DNS server - dnsServer := dnsCfg.Servers[0] - if !strings.Contains(dnsServer, ":") { - dnsServer += ":53" - } - - return NewClient(dialer, dnsServer), nil -} - -func NewClient(dialer *net.Dialer, dnsServer string) *Client { - return &Client{ - client: &dns.Client{ - Net: "udp", - Dialer: dialer, - DialTimeout: DialTimeout, - ReadTimeout: ReadTimeout, - WriteTimeout: WriteTimeout, - }, - dnsServer: dnsServer, - } -} - -func (c *Client) DNSServer() string { - return c.dnsServer -} - -func createDNSMessage(host string, typ uint16) *dns.Msg { - msg := &dns.Msg{} - msg.SetQuestion(dns.Fqdn(host), typ) - msg.Authoritative = true - // Disable recursion because we want to know if service pod is available in k8s. - // No need to recursively ask other servers. Also disables caching which we also want. - msg.RecursionDesired = false - msg.RecursionAvailable = false - return msg -} - -func (c *Client) Resolve(ctx context.Context, host string) (string, error) { - msg := createDNSMessage(host, dns.TypeA) - - // Send DNS query - resp, _, err := c.client.ExchangeContext(ctx, msg, c.dnsServer) - if err != nil { - return "", err - } - - if len(resp.Answer) == 0 { - msg = createDNSMessage(host, dns.TypeAAAA) - resp, _, err = c.client.ExchangeContext(ctx, msg, c.dnsServer) - if err != nil { - return "", err - } - - if len(resp.Answer) > 0 { - // nolint: gosec // we don't need to use crypto.rand here - ip := resp.Answer[rand.Intn(len(resp.Answer))].(*dns.AAAA).AAAA.String() - return ip, nil - } - - return "", &net.DNSError{ - Err: fmt.Sprintf(`host not found: %s`, host), - Name: host, - Server: c.dnsServer, - IsNotFound: true, - } - } - - // nolint: gosec // we don't need to use crypto.rand here - ip := resp.Answer[rand.Intn(len(resp.Answer))].(*dns.A).A.String() - return ip, nil -} diff --git a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock.go b/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock.go deleted file mode 100644 index 2f9a58d27e..0000000000 --- a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock.go +++ /dev/null @@ -1,186 +0,0 @@ -// Based on https://github.com/kuritka/go-fake-dns -// License: MIT - -package dnsmock - -import ( - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/miekg/dns" - - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" -) - -type DNSRecordError struct { - Err error -} - -func (d DNSRecordError) Error() string { - return d.Err.Error() -} - -// Server acts as DNS server but returns mock values. -type Server struct { - server *dns.Server - records map[uint16][]dns.RR - updateLock *sync.Mutex -} - -func New(port int) *Server { - server := &Server{ - records: make(map[uint16][]dns.RR), - updateLock: &sync.Mutex{}, - } - - var handler dns.HandlerFunc = server.handleRequest - - server.server = &dns.Server{ - Addr: "[::]:" + strconv.FormatInt(int64(port), 10), - Net: "udp", - Handler: handler, - } - - return server -} - -func (s *Server) Start() error { - startedCh := make(chan struct{}) - s.server.NotifyStartedFunc = func() { - close(startedCh) - } - - serverErrCh := make(chan error, 1) - go func() { - serverErrCh <- s.server.ListenAndServe() - }() - - // Wait for DNS server startup - select { - case <-startedCh: - // ok - case err := <-serverErrCh: - return err - case <-time.After(5 * time.Second): - return errors.New("DNS server start timeout") - } - - return nil -} - -func (s *Server) Addr() string { - return s.server.PacketConn.LocalAddr().String() -} - -func (s *Server) Shutdown() error { - return s.server.Shutdown() -} - -func (s *Server) AddTXTRecord(fqdn string, strings ...string) { - t := &dns.TXT{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: 0}, - Txt: strings, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeTXT] = append(s.records[dns.TypeTXT], t) -} - -func (s *Server) AddNSRecord(fqdn, nsName string) { - ns := &dns.NS{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeNS, Class: dns.ClassINET, Ttl: 0}, - Ns: nsName, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeNS] = append(s.records[dns.TypeNS], ns) -} - -func (s *Server) AddAOrAAAARecord(fqdn string, ip net.IP) error { - ipv4 := ip.To4() - if ipv4 != nil { - return s.AddARecord(fqdn, ipv4) - } - - return s.AddAAAARecord(fqdn, ip) -} - -func (s *Server) AddARecord(fqdn string, ip net.IP) error { - ipv4 := ip.To4() - if ipv4 == nil { - return &DNSRecordError{Err: errors.New("Unable to create A record")} - } - - rr := &dns.A{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0}, - A: ipv4, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeA] = append(s.records[dns.TypeA], rr) - return nil -} - -func (s *Server) AddAAAARecord(fqdn string, ip net.IP) error { - ipv6 := ip.To16() - if ipv6 == nil { - return &DNSRecordError{Err: errors.New("Unable to create AAAA record")} - } - - rr := &dns.AAAA{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 0}, - AAAA: ipv6, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeAAAA] = append(s.records[dns.TypeAAAA], rr) - return nil -} - -func (s *Server) RemoveTXTRecords(fqdn string) { - s.removeRecords(dns.TypeTXT, fqdn) -} - -func (s *Server) RemoveNSRecords(fqdn string) { - s.removeRecords(dns.TypeNS, fqdn) -} - -func (s *Server) RemoveARecords(fqdn string) { - s.removeRecords(dns.TypeA, fqdn) -} - -func (s *Server) RemoveAAAARecords(fqdn string) { - s.removeRecords(dns.TypeAAAA, fqdn) -} - -func (s *Server) removeRecords(dnsType uint16, fqdn string) { - s.updateLock.Lock() - defer s.updateLock.Unlock() - records := []dns.RR{} - for _, rr := range s.records[dnsType] { - if rr.Header().Name != fqdn { - records = append(records, rr) - } - } - s.records[dnsType] = records -} - -func (s *Server) handleRequest(w dns.ResponseWriter, r *dns.Msg) { - msg := new(dns.Msg) - msg.SetReply(r) - msg.Compress = false - s.updateLock.Lock() - defer s.updateLock.Unlock() - if s.records[r.Question[0].Qtype] != nil { - for _, rr := range s.records[r.Question[0].Qtype] { - fqdn := strings.Split(rr.String(), "\t")[0] - if fqdn == r.Question[0].Name { - msg.Answer = append(msg.Answer, rr) - } - } - } - _ = w.WriteMsg(msg) -} diff --git a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock_test.go b/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock_test.go deleted file mode 100644 index f5a88db812..0000000000 --- a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Based on https://github.com/kuritka/go-fake-dns -// License: MIT - -package dnsmock_test - -import ( - "net" - "testing" - - "github.com/miekg/dns" - "github.com/stretchr/testify/require" - - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" -) - -func TestMultipleTXTRecords(t *testing.T) { - t.Parallel() - - dnsMock := dnsmock.New(0) - dnsMock.AddTXTRecord("heartbeat-us.cloud.example.com.", "1") - dnsMock.AddTXTRecord("heartbeat-uk.cloud.example.com.", "2") - dnsMock.AddTXTRecord("heartbeat-eu.cloud.example.com.", "0", "6", "8") - err := dnsMock.Start() - require.NoError(t, err) - - defer dnsMock.Shutdown() - - g := new(dns.Msg) - g.SetQuestion("ip.blah.cloud.example.com.", dns.TypeTXT) - a, err := dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) - - g = new(dns.Msg) - g.SetQuestion("heartbeat-uk.cloud.example.com.", dns.TypeTXT) - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Len(t, a.Answer, 1) - require.Len(t, a.Answer[0].(*dns.TXT).Txt, 1) - require.Equal(t, "2", a.Answer[0].(*dns.TXT).Txt[0]) - - g = new(dns.Msg) - g.SetQuestion("heartbeat-eu.cloud.example.com.", dns.TypeTXT) - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Len(t, a.Answer, 1) - require.Len(t, a.Answer[0].(*dns.TXT).Txt, 3) - require.Equal(t, "0", a.Answer[0].(*dns.TXT).Txt[0]) - require.Equal(t, "6", a.Answer[0].(*dns.TXT).Txt[1]) - require.Equal(t, "8", a.Answer[0].(*dns.TXT).Txt[2]) -} - -func TestSimple(t *testing.T) { - t.Parallel() - - dnsMock := dnsmock.New(0) - dnsMock.AddNSRecord("blah.cloud.example.com.", "gslb-ns-us-cloud.example.com.") - dnsMock.AddNSRecord("blah.cloud.example.com.", "gslb-ns-uk-cloud.example.com.") - dnsMock.AddNSRecord("blah.cloud.example.com.", "gslb-ns-eu-cloud.example.com.") - dnsMock.AddTXTRecord("First", "Second", "Banana") - dnsMock.AddTXTRecord("White", "Red", "Purple") - dnsMock.AddARecord("ip.blah.cloud.example.com.", net.IPv4(10, 0, 1, 5)) - err := dnsMock.Start() - require.NoError(t, err) - - defer dnsMock.Shutdown() - - g := new(dns.Msg) - g.SetQuestion("ip.blah.cloud.example.com.", dns.TypeA) - a, err := dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.NotEmpty(t, a.Answer) - - dnsMock.RemoveARecords("ip.blah.cloud.example.com.") - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) -} - -func TestWrongARecord(t *testing.T) { - t.Parallel() - - dnsMock := dnsmock.New(0) - require.Error(t, dnsMock.AddARecord("ipv4.net.", net.IPv6loopback)) - require.Error(t, dnsMock.AddAAAARecord("ipv6.net.", net.IP{0, 0})) - err := dnsMock.Start() - require.NoError(t, err) - - defer dnsMock.Shutdown() - - g := new(dns.Msg) - g.SetQuestion("ipv4.net.", dns.TypeA) - a, err := dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) - - g.SetQuestion("ipv6.net.", dns.TypeAAAA) - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) -} diff --git a/internal/pkg/service/appsproxy/proxy/transport/transport.go b/internal/pkg/service/appsproxy/proxy/transport/transport.go index d3222f4206..d290688744 100644 --- a/internal/pkg/service/appsproxy/proxy/transport/transport.go +++ b/internal/pkg/service/appsproxy/proxy/transport/transport.go @@ -1,21 +1,14 @@ -// Package transport provides HTTP transport for the reverse HTTP proxy with custom DNS resolving. -// DNS client is used to fast and reliable determine if the target data app is running. +// Package transport provides HTTP transport for the reverse HTTP proxy. package transport import ( - "context" - "net" "net/http" - "net/http/httptrace" "time" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/otel/attribute" "golang.org/x/net/http2" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" ) // TLSHandshakeTimeout specifies the default timeout of TLS handshake. @@ -39,96 +32,17 @@ const HTTP2PingTimeout = 2 * time.Second // HTTP2WriteByteTimeout is the timeout after which the connection will be closed no data can be written to it. const HTTP2WriteByteTimeout = 15 * time.Second -const DNSResolveTimeout = 5 * time.Second - type dependencies interface { Telemetry() telemetry.Telemetry } // New creates new http transport intended to be used with NewSingleHostReverseProxy. -// DNS server address is loaded from the "/etc/resolv.conf". -func New(d dependencies, dnsServer string) (http.RoundTripper, error) { - return NewWithDNSServer(d, dnsServer) -} - -// NewWithDNSServer creates new http transport intended to be used with NewSingleHostReverseProxy. -func NewWithDNSServer(d dependencies, dnsServerAddress string) (http.RoundTripper, error) { +func New(d dependencies) (http.RoundTripper, error) { dialer := newDialer() - var dnsClient *dns.Client - var err error - if dnsServerAddress == "" { - dnsClient, err = dns.NewClientFromEtc(dialer) - if err != nil { - return nil, err - } - } else { - dnsClient = dns.NewClient(dialer, dnsServerAddress) - } - - tel := d.Telemetry() - - dialContext := func(ctx context.Context, network, address string) (net.Conn, error) { - host, port, err := net.SplitHostPort(address) - if err != nil { - return nil, err - } - - if net.ParseIP(host) != nil { - return dialer.DialContext(ctx, network, address) - } - - // Get tracing hooks - trace := httptrace.ContextClientTrace(ctx) - - // Trace DNSStart - ctx, dnsSpan := tel.Tracer().Start(ctx, "keboola.go.apps-proxy.transport.dns.resolve") - dnsSpan.SetAttributes( - attribute.String("transport.dns.resolve.host", host), - attribute.String("transport.dns.resolve.server", dnsClient.DNSServer()), - ) - if trace != nil && trace.DNSStart != nil { - trace.DNSStart(httptrace.DNSStartInfo{Host: host}) - } - - // Create context for DNS resolving - // It separates the events/tracing of the connection to the DNS server, from the connection to the target server. - resolveCtx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), DNSResolveTimeout, errors.New("DNS resolve timeout")) - defer cancel() - - // We are using custom DNS resolving to detect if the target host - data app, is running. - // The DNS query is not recursive and not cached, see "dns" package for details. - ip, err := dnsClient.Resolve(resolveCtx, host) //nolint:contextcheck - - // Trace DNSDone - if trace != nil && trace.DNSDone != nil { - trace.DNSDone(httptrace.DNSDoneInfo{ - Addrs: []net.IPAddr{{IP: net.ParseIP(ip)}}, - Err: err, - }) - } - - // Handle DNS error - dnsSpan.End(&err) - if err != nil { - return nil, err - } - - // Dial - ctx, dialSpan := tel.Tracer().Start(ctx, "keboola.go.apps-proxy.transport.dial") - dialSpan.SetAttributes( - attribute.String("transport.dial.network", network), - attribute.String("transport.dial.ip", ip), - attribute.String("transport.dial.port", port), - ) - conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip, port)) - dialSpan.End(&err) - return conn, err - } - httpTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, - DialContext: dialContext, + DialContext: dialer.DialContext, ForceAttemptHTTP2: true, // HTTP2 is preferred. DisableKeepAlives: false, TLSHandshakeTimeout: TLSHandshakeTimeout, From bea114f70eaf73d8a40472812ed90881e6bfaede Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Tue, 3 Mar 2026 14:24:53 +0100 Subject: [PATCH 08/11] fix: wait for informer cache sync before creating K8s objects in proxy tests Add WaitForCacheSync method to StateWatcher so tests can block until the informer has completed its initial list-watch. This ensures the watch is established before test objects are created, preventing a race where the 5-second timeout in registerDefaultK8sApps was exhausted under heavy parallel test load. --- .../appsproxy/dataapps/k8sapp/watcher.go | 19 +++++++++++++++---- .../pkg/service/appsproxy/proxy/proxy_test.go | 2 ++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go index eb582cc9b4..aa84232c99 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go @@ -32,10 +32,11 @@ type entry struct { // StateWatcher watches App CRDs in Kubernetes and provides a local cache of app states. type StateWatcher struct { - client dynamic.Interface - namespace string - logger log.Logger - serviceURLBuilder func(name string) string + client dynamic.Interface + namespace string + logger log.Logger + serviceURLBuilder func(name string) string + hasSynced cache.InformerSynced // byName: K8s object name → AppID byName sync.Map // byAppID: AppID → entry{k8sName, state} @@ -73,6 +74,7 @@ func NewStateWatcher(d dependencies, client dynamic.Interface, namespace string) serviceURLBuilder: func(name string) string { return fmt.Sprintf("http://%s.%s.svc.cluster.local:8888", name, namespace) }, + hasSynced: func() bool { return false }, } ctx, cancel := context.WithCancel(context.Background()) @@ -112,6 +114,8 @@ func NewStateWatcher(d dependencies, client dynamic.Interface, namespace string) w.logger.Errorf(ctx, "failed to add event handler to App informer: %s", err) } + w.hasSynced = informer.HasSynced + go informer.Run(ctx.Done()) // Log when the cache has synced so operators know the watcher is ready. @@ -144,6 +148,13 @@ func (w *StateWatcher) SetServiceURLBuilder(f func(name string) string) { w.serviceURLBuilder = f } +// WaitForCacheSync blocks until the informer cache has completed its initial list, +// the stopCh is closed, or the context is cancelled. +// Intended for use in tests to ensure the watch is established before creating objects. +func (w *StateWatcher) WaitForCacheSync(ctx context.Context) bool { + return cache.WaitForCacheSync(ctx.Done(), w.hasSynced) +} + // SetDesiredRunning patches .spec.state = "Running" on the App CRD for the given appID. // If the appID is not yet in the cache, no patch is sent. func (w *StateWatcher) SetDesiredRunning(ctx context.Context, appID api.AppID) error { diff --git a/internal/pkg/service/appsproxy/proxy/proxy_test.go b/internal/pkg/service/appsproxy/proxy/proxy_test.go index 273d296d2a..eb3955f467 100644 --- a/internal/pkg/service/appsproxy/proxy/proxy_test.go +++ b/internal/pkg/service/appsproxy/proxy/proxy_test.go @@ -2875,6 +2875,8 @@ func registerDefaultK8sApps(t *testing.T, apps []api.AppConfig, fakeClient *k8sf t.Helper() // Override URL builder so requests route to the test server instead of a real K8s service. watcher.SetServiceURLBuilder(func(_ string) string { return serviceURL }) + // Wait for the informer to complete its initial list so the watch is established before we create objects. + require.True(t, watcher.WaitForCacheSync(t.Context()), "App CRD informer cache sync timed out") for _, app := range apps { obj := &unstructured.Unstructured{ Object: map[string]any{ From 8837c5e98ac82dcd7c287430a3c00288fe733a28 Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Tue, 3 Mar 2026 14:25:11 +0100 Subject: [PATCH 09/11] ci: exclude deprecated keboola.docs.apiary.io from lychee link check The Apiary-hosted API docs are no longer maintained and return 502. The links are in pre-existing documentation files; exclude the domain from the link checker rather than removing the historical references. --- .github/workflows/test-lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml index 8294e96255..737da38b67 100644 --- a/.github/workflows/test-lint.yml +++ b/.github/workflows/test-lint.yml @@ -51,6 +51,7 @@ jobs: --exclude '^https://community.chocolatey.org/.*' --exclude '^https://packages.debian.org/$' --exclude '^https://test.hub.keboola.local/$' + --exclude '^https://keboola.docs.apiary.io/.*' - name: Run code linters run: task lint From d9c4dc51dd01501cf7a2f19d3103ee542c9c2723 Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Tue, 3 Mar 2026 14:40:24 +0100 Subject: [PATCH 10/11] refactor: read upstream URL from .status.appsProxy.upstreamUrl in App CRD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace .status.appsProxyServiceRef.name (which was converted to a URL via serviceURLBuilder) with a new .status.appsProxy.upstreamUrl field that holds the fully-qualified upstream URL directly. Remove the serviceURLBuilder indirection from StateWatcher — the URL is now parsed directly from the CRD status field. --- go.mod | 4 +-- .../appsproxy/dataapps/k8sapp/types.go | 10 +++---- .../appsproxy/dataapps/k8sapp/watcher.go | 26 +++++-------------- .../appsproxy/dataapps/k8sapp/watcher_test.go | 12 ++++----- .../appsproxy/proxy/apphandler/manager.go | 4 +-- .../proxy/apphandler/upstream/upstream.go | 8 +++--- .../pkg/service/appsproxy/proxy/proxy_test.go | 10 +++---- 7 files changed, 30 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 35ec207eea..a5f153de88 100644 --- a/go.mod +++ b/go.mod @@ -95,6 +95,7 @@ require ( golang.org/x/sync v0.19.0 google.golang.org/grpc v1.78.0 gopkg.in/Knetic/govaluate.v3 v3.0.0 + k8s.io/client-go v0.33.3 v.io/x/lib v0.1.21 ) @@ -141,7 +142,6 @@ require ( golang.org/x/tools/godoc v0.1.0-deprecated // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - k8s.io/client-go v0.33.3 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect @@ -400,7 +400,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gotest.tools/gotestsum v1.13.0 // indirect - k8s.io/apimachinery v0.33.3 // indirect + k8s.io/apimachinery v0.33.3 k8s.io/klog/v2 v2.130.1 // indirect mvdan.cc/gofumpt v0.9.2 // indirect ) diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go index d7112736b8..2ec1d8d70b 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go @@ -41,16 +41,16 @@ type appSpec struct { type AppInfo struct { ActualState AppActualState AutoRestartEnabled bool - // UpstreamTarget is the pre-parsed URL from .status.appsProxyServiceRef. + // UpstreamTarget is the pre-parsed URL from .status.appsProxy.upstreamUrl. // Nil when the field is absent or unparseable. UpstreamTarget *url.URL } type appStatus struct { - CurrentState AppActualState `json:"currentState"` - AppsProxyServiceRef appsProxyServiceRef `json:"appsProxyServiceRef,omitempty"` + CurrentState AppActualState `json:"currentState"` + AppsProxy appsProxy `json:"appsProxy,omitempty"` } -type appsProxyServiceRef struct { - Name string `json:"name,omitempty"` +type appsProxy struct { + UpstreamURL string `json:"upstreamUrl,omitempty"` } diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go index aa84232c99..d556168788 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go @@ -3,7 +3,6 @@ package k8sapp import ( "context" "encoding/json" - "fmt" "net/url" "sync" @@ -27,16 +26,15 @@ type entry struct { k8sName string state AppActualState autoRestartEnabled bool - upstreamTarget *url.URL // pre-parsed; nil when appsProxyServiceRef absent/invalid + upstreamTarget *url.URL // pre-parsed; nil when appsProxy.upstreamUrl absent/invalid } // StateWatcher watches App CRDs in Kubernetes and provides a local cache of app states. type StateWatcher struct { - client dynamic.Interface - namespace string - logger log.Logger - serviceURLBuilder func(name string) string - hasSynced cache.InformerSynced + client dynamic.Interface + namespace string + logger log.Logger + hasSynced cache.InformerSynced // byName: K8s object name → AppID byName sync.Map // byAppID: AppID → entry{k8sName, state} @@ -71,9 +69,6 @@ func NewStateWatcher(d dependencies, client dynamic.Interface, namespace string) client: client, namespace: namespace, logger: d.Logger().WithComponent("k8sapp.watcher"), - serviceURLBuilder: func(name string) string { - return fmt.Sprintf("http://%s.%s.svc.cluster.local:8888", name, namespace) - }, hasSynced: func() bool { return false }, } @@ -142,12 +137,6 @@ func (w *StateWatcher) GetState(appID api.AppID) (AppInfo, bool) { }, true } -// SetServiceURLBuilder overrides the function used to construct the upstream URL from a service name. -// It is intended for use in tests only, to point the proxy at a local test server. -func (w *StateWatcher) SetServiceURLBuilder(f func(name string) string) { - w.serviceURLBuilder = f -} - // WaitForCacheSync blocks until the informer cache has completed its initial list, // the stopCh is closed, or the context is cancelled. // Intended for use in tests to ensure the watch is established before creating objects. @@ -213,12 +202,11 @@ func (w *StateWatcher) handleUpsert(ctx context.Context, obj any) { } var upstreamTarget *url.URL - if name := appObj.Status.AppsProxyServiceRef.Name; name != "" { - rawURL := w.serviceURLBuilder(name) + if rawURL := appObj.Status.AppsProxy.UpstreamURL; rawURL != "" { if t, err := url.Parse(rawURL); err == nil { upstreamTarget = t } else { - w.logger.Warnf(ctx, "App CRD %q (appID=%s) invalid upstream URL %q for appsProxyServiceRef %q: %s", k8sName, appObj.Spec.AppID, rawURL, name, err) + w.logger.Warnf(ctx, "App CRD %q (appID=%s) invalid upstream URL %q from appsProxy.upstreamUrl: %s", k8sName, appObj.Spec.AppID, rawURL, err) } } diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go index 0c16d8d1d0..d19be19aa5 100644 --- a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go @@ -71,10 +71,10 @@ func newAppObject(k8sName, appID string, state k8sapp.AppActualState) *unstructu } } -// newAppObjectWithServiceRef creates an unstructured App CRD object with appsProxyServiceRef set. -func newAppObjectWithServiceRef(k8sName, appID string, state k8sapp.AppActualState, serviceName string) *unstructured.Unstructured { +// newAppObjectWithUpstreamURL creates an unstructured App CRD object with appsProxy.upstreamUrl set. +func newAppObjectWithUpstreamURL(k8sName, appID string, state k8sapp.AppActualState, upstreamURL string) *unstructured.Unstructured { obj := newAppObject(k8sName, appID, state) - obj.Object["status"].(map[string]any)["appsProxyServiceRef"] = map[string]any{"name": serviceName} + obj.Object["status"].(map[string]any)["appsProxy"] = map[string]any{"upstreamUrl": upstreamURL} return obj } @@ -167,7 +167,7 @@ func TestStateWatcher_GetState_UpstreamTarget(t *testing.T) { fakeClient := newFakeClient() d := newTestDeps(t) - appObj := newAppObjectWithServiceRef("my-app-k8s", "app-123", k8sapp.AppActualStateRunning, "svc-name") + appObj := newAppObjectWithUpstreamURL("my-app-k8s", "app-123", k8sapp.AppActualStateRunning, "http://my-svc.keboola.svc.cluster.local:8888") _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( t.Context(), appObj, metav1.CreateOptions{}, ) @@ -184,7 +184,7 @@ func TestStateWatcher_GetState_UpstreamTarget(t *testing.T) { require.NotNil(t, info.UpstreamTarget) assert.Equal(t, "http", info.UpstreamTarget.Scheme) - assert.Equal(t, "svc-name.keboola.svc.cluster.local:8888", info.UpstreamTarget.Host) + assert.Equal(t, "my-svc.keboola.svc.cluster.local:8888", info.UpstreamTarget.Host) } func TestStateWatcher_GetState_UpstreamTarget_AbsentWhenMissing(t *testing.T) { @@ -193,7 +193,7 @@ func TestStateWatcher_GetState_UpstreamTarget_AbsentWhenMissing(t *testing.T) { fakeClient := newFakeClient() d := newTestDeps(t) - // App CRD without appsProxyServiceRef. + // App CRD without appsProxy.upstreamUrl. appObj := newAppObject("my-app-k8s", "app-123", k8sapp.AppActualStateRunning) _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( t.Context(), appObj, metav1.CreateOptions{}, diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/manager.go b/internal/pkg/service/appsproxy/proxy/apphandler/manager.go index 3c07033710..d72ca9daa5 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/manager.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/manager.go @@ -32,7 +32,7 @@ type appHandlerWrapper struct { lock *sync.Mutex handler http.Handler cancel context.CancelCauseFunc - serviceRef string // appsProxyServiceRef in effect when this handler was created + serviceRef string // appsProxy.upstreamUrl in effect when this handler was created } type dependencies interface { @@ -71,7 +71,7 @@ func (m *Manager) HandlerFor(ctx context.Context, result appconfig.AppConfigResu return m.newErrorHandler(ctx, api.AppConfig{ID: result.AppID}, result.Err) } - // Create a new handler, if needed (config changed or appsProxyServiceRef changed) + // Create a new handler, if needed (config changed or appsProxy.upstreamUrl changed) currentRef := m.upstreamManager.CurrentServiceRef(result.AppID) if wrapper.handler == nil || result.Modified || wrapper.serviceRef != currentRef { if wrapper.cancel != nil { diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index 4f525d1848..2f96abc4a0 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -52,7 +52,7 @@ type Manager struct { type AppUpstream struct { manager *Manager app api.AppConfig - target *url.URL // parsed from appsProxyServiceRef at creation; nil when absent + target *url.URL // parsed from appsProxy.upstreamUrl at creation; nil when absent handler *chain.Chain wsHandler *chain.Chain cancelWs context.CancelCauseFunc @@ -98,7 +98,7 @@ func (m *Manager) Shutdown(ctx context.Context) { m.wg.Wait() } -// CurrentServiceRef returns the upstream URL string for appID from the K8s cache. +// CurrentServiceRef returns the appsProxy.upstreamUrl string for appID from the K8s cache. // Returns "" when the app is not cached or the field is absent/invalid. func (m *Manager) CurrentServiceRef(appID api.AppID) string { info, ok := m.stateWatcher.GetState(appID) @@ -169,9 +169,9 @@ func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request return nil } - // Target set at creation time; nil means appsProxyServiceRef was absent. + // Target set at creation time; nil means appsProxy.upstreamUrl was absent. if u.target == nil { - u.manager.logger.Debugf(ctx, "app %q has no appsProxyServiceRef, serving spinner page", u.app.ID) + u.manager.logger.Debugf(ctx, "app %q has no appsProxy.upstreamUrl, serving spinner page", u.app.ID) u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) return nil } diff --git a/internal/pkg/service/appsproxy/proxy/proxy_test.go b/internal/pkg/service/appsproxy/proxy/proxy_test.go index eb3955f467..eca265c6cc 100644 --- a/internal/pkg/service/appsproxy/proxy/proxy_test.go +++ b/internal/pkg/service/appsproxy/proxy/proxy_test.go @@ -2461,7 +2461,7 @@ func TestAppProxyRouter(t *testing.T) { client := createHTTPClient(t, proxyURL) - // Default K8s setup: register all test apps as Running with appsProxyServiceRef. + // Default K8s setup: register all test apps as Running with appsProxy.upstreamUrl. registerDefaultK8sApps(t, apps, mocked.TestFakeK8sClient(), d.AppStateWatcher(), appURL.String()) if tc.setupK8s != nil { tc.setupK8s(t, mocked.TestFakeK8sClient(), d.AppStateWatcher()) @@ -2868,13 +2868,11 @@ func htmlLinkTo(url string) string { } // registerDefaultK8sApps creates Running App CRD objects in the fake K8s client for all test apps -// and waits for the StateWatcher to sync them. The proxy reads appsProxyServiceRef from the CRD +// and waits for the StateWatcher to sync them. The proxy reads appsProxy.upstreamUrl from the CRD // to get the upstream URL, so this is required for requests to route to the upstream. // Must be called before tc.setupK8s so per-test overrides (Patch) can override specific apps. func registerDefaultK8sApps(t *testing.T, apps []api.AppConfig, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher, serviceURL string) { t.Helper() - // Override URL builder so requests route to the test server instead of a real K8s service. - watcher.SetServiceURLBuilder(func(_ string) string { return serviceURL }) // Wait for the informer to complete its initial list so the watch is established before we create objects. require.True(t, watcher.WaitForCacheSync(t.Context()), "App CRD informer cache sync timed out") for _, app := range apps { @@ -2891,8 +2889,8 @@ func registerDefaultK8sApps(t *testing.T, apps []api.AppConfig, fakeClient *k8sf }, "status": map[string]any{ "currentState": string(k8sapp.AppActualStateRunning), - "appsProxyServiceRef": map[string]any{ - "name": "app-" + string(app.ID), + "appsProxy": map[string]any{ + "upstreamUrl": serviceURL, }, }, }, From a5b8f583ad6d5b914251f6a55b2c0f0eed058b51 Mon Sep 17 00:00:00 2001 From: Pepa Martinec Date: Tue, 3 Mar 2026 15:16:50 +0100 Subject: [PATCH 11/11] fix: clear Host header when forwarding requests to upstream httputil.NewSingleHostReverseProxy rewrites req.URL.Host but leaves req.Host (the actual Host header) set to the original incoming value. Upstreams that route by Host header would then receive the proxy's public hostname instead of the upstream target hostname. Clear req.Host in the Director so Go's HTTP client derives the Host header from req.URL.Host instead. --- .../proxy/apphandler/upstream/upstream.go | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index 2f96abc4a0..577c5ed807 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -183,11 +183,26 @@ func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request return u.handler.ServeHTTPOrError(rw, req) } -func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { +func (u *AppUpstream) newReverseProxy() *httputil.ReverseProxy { proxy := httputil.NewSingleHostReverseProxy(u.target) proxy.Transport = u.manager.transport proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app) + // Clear req.Host so Go's HTTP client derives the Host header from req.URL.Host. + // httputil.NewSingleHostReverseProxy rewrites req.URL.Host but leaves req.Host + // (the actual Host header) set to the original incoming value, which causes + // upstreams that route by Host to return wrong responses. + origDirector := proxy.Director + proxy.Director = func(req *http.Request) { + origDirector(req) + req.Host = "" + } + return proxy +} + +func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { + proxy := u.newReverseProxy() + return chain. New(chain.HandlerFunc(func(w http.ResponseWriter, req *http.Request) error { ctx := ctxattr.ContextWith(req.Context(), attribute.Bool(attrWebsocket, false)) @@ -203,9 +218,7 @@ func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { } func (u *AppUpstream) newWebsocketProxy(timeout time.Duration) *chain.Chain { - proxy := httputil.NewSingleHostReverseProxy(u.target) - proxy.Transport = u.manager.transport - proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app) + proxy := u.newReverseProxy() return chain. New(chain.HandlerFunc(func(w http.ResponseWriter, req *http.Request) error {