diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml index 8294e96255..737da38b67 100644 --- a/.github/workflows/test-lint.yml +++ b/.github/workflows/test-lint.yml @@ -51,6 +51,7 @@ jobs: --exclude '^https://community.chocolatey.org/.*' --exclude '^https://packages.debian.org/$' --exclude '^https://test.hub.keboola.local/$' + --exclude '^https://keboola.docs.apiary.io/.*' - name: Run code linters run: task lint diff --git a/go.mod b/go.mod index 8a39ebefe8..a5f153de88 100644 --- a/go.mod +++ b/go.mod @@ -95,6 +95,7 @@ require ( golang.org/x/sync v0.19.0 google.golang.org/grpc v1.78.0 gopkg.in/Knetic/govaluate.v3 v3.0.0 + k8s.io/client-go v0.33.3 v.io/x/lib v0.1.21 ) @@ -108,10 +109,16 @@ require ( github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect github.com/cheggaaa/pb/v3 v3.1.6 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-test/deep v1.1.1 // indirect + github.com/google/gnostic-models v0.6.9 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/icholy/gomajor v0.15.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/minio/simdjson-go v0.4.5 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/otiai10/mint v1.6.3 // indirect @@ -120,6 +127,7 @@ require ( github.com/shirou/gopsutil/v4 v4.25.8-0.20250809033336-ffcdc2b7662f // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/theckman/httpforwarded v0.4.0 // indirect + github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/etcdctl/v3 v3.6.7 // indirect go.etcd.io/gofail v0.2.0 // indirect go.etcd.io/raft/v3 v3.6.0 // indirect @@ -132,7 +140,14 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/tools/cmd/godoc v0.1.0-deprecated // indirect golang.org/x/tools/godoc v0.1.0-deprecated // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) require ( @@ -146,7 +161,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/iancoleman/strcase v0.3.0 github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -385,7 +400,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gotest.tools/gotestsum v1.13.0 // indirect - k8s.io/apimachinery v0.33.3 // indirect + k8s.io/apimachinery v0.33.3 k8s.io/klog/v2 v2.130.1 // indirect mvdan.cc/gofumpt v0.9.2 // indirect ) diff --git a/go.sum b/go.sum index 72f919e9eb..8c542ed0b4 100644 --- a/go.sum +++ b/go.sum @@ -287,6 +287,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -352,6 +353,10 @@ github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhs github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -380,8 +385,12 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/inflect v0.21.2 h1:0gClGlGcxifcJR56zwvhaOulnNgnhc4qTAkob5ObnSM= github.com/go-openapi/inflect v0.21.2/go.mod h1:INezMuUu7SJQc2AyR3WO0DqqYUJSj8Kb4hBd7WtjlAw= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-ozzo/ozzo-routing v2.1.4+incompatible h1:gQmNyAwMnBHr53Nma2gPTfVVc6i2BuAwCWPam2hIvKI= @@ -467,6 +476,10 @@ github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60 github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -516,6 +529,8 @@ github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWS github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= @@ -608,6 +623,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -636,6 +652,7 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/makeworld-the-better-one/dither/v2 v2.4.0 h1:Az/dYXiTcwcRSe59Hzw4RI1rSnAZns+1msaCXetrMFE= @@ -726,9 +743,11 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.133.0 h1:iPei+89a2EK4LuN4HeIRzZNE6XxCyrKfBKG3BkK/ViU= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.133.0/go.mod h1:asV77TgnGfc7A+a9jggdsnlLlW5dnJT8RroVuf5slko= github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.133.0 h1:4ca2pM3+xDMB9H3UnhjAiNg7EpIydZ7HdohOexU8xb8= @@ -919,6 +938,8 @@ github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIj github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28= github.com/writeas/go-strip-markdown/v2 v2.1.1/go.mod h1:UvvgPJgn1vvN8nWuE5e7v/+qmDu3BSVnKAB6Gl7hFzA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= @@ -1270,7 +1291,13 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -1298,14 +1325,38 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8= +k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY= rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= +sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= v.io/x/lib v0.1.21 h1:PtTlthjCNjjdfZviHr2hDhGSLqlyOTxSaKrBMSZOj4Q= diff --git a/internal/pkg/service/appsproxy/config/config.go b/internal/pkg/service/appsproxy/config/config.go index 7ec6ab86cb..1d9dc41b03 100644 --- a/internal/pkg/service/appsproxy/config/config.go +++ b/internal/pkg/service/appsproxy/config/config.go @@ -19,12 +19,12 @@ type Config struct { PProf pprof.Config `configKey:"pprof"` Datadog datadog.Config `configKey:"datadog"` Metrics prometheus.Config `configKey:"metrics"` - DNSServer string `configKey:"dnsServer" configUsage:"DNS server for proxy. If empty, the /etc/resolv.conf is used."` API API `configKey:"api"` CookieSecretSalt string `configKey:"cookieSecretSalt" configUsage:"Cookie secret needed by OAuth 2 Proxy." validate:"required" sensitive:"true"` Upstream Upstream `configKey:"-" configUsage:"Configuration options for upstream"` SandboxesAPI SandboxesAPI `configKey:"sandboxesAPI"` CsrfTokenSalt string `configKey:"csrfTokenSalt" configUsage:"Salt used for generating CSRF tokens" validate:"required" sensitive:"true"` + K8s K8s `configKey:"k8s"` } type API struct { @@ -42,6 +42,11 @@ type Upstream struct { WsTimeout time.Duration `configKey:"wsTimeout" configUsage:"Timeout for websocket request on upstream"` } +type K8s struct { + AppsNamespace string `configKey:"appsNamespace" configUsage:"Kubernetes namespace where apps (App CRDs) run." validate:"required"` + Kubeconfig string `configKey:"kubeconfig" configUsage:"Path to kubeconfig file. Uses in-cluster config if empty."` +} + func New() Config { return Config{ DebugLog: false, diff --git a/internal/pkg/service/appsproxy/dataapps/api/error.go b/internal/pkg/service/appsproxy/dataapps/api/error.go index 638c9bf4cd..c57bc7ca4e 100644 --- a/internal/pkg/service/appsproxy/dataapps/api/error.go +++ b/internal/pkg/service/appsproxy/dataapps/api/error.go @@ -5,10 +5,6 @@ import ( "net/http" ) -const ( - ContextCodeRestartDisabled = "apps.restartDisabled" -) - // Error represents the structure of Sandboxes API error. type Error struct { Message string `json:"error"` @@ -37,14 +33,6 @@ func (e *Error) ErrorExceptionID() string { return e.ExceptionID } -func (e *Error) HasRestartDisabled() bool { - if e.Context == nil { - return false - } - contextCode, ok := e.Context["code"].(string) - return ok && contextCode == ContextCodeRestartDisabled -} - // StatusCode returns HTTP status code. func (e *Error) StatusCode() int { return e.response.StatusCode diff --git a/internal/pkg/service/appsproxy/dataapps/api/wakeup.go b/internal/pkg/service/appsproxy/dataapps/api/wakeup.go deleted file mode 100644 index 4d26154538..0000000000 --- a/internal/pkg/service/appsproxy/dataapps/api/wakeup.go +++ /dev/null @@ -1,32 +0,0 @@ -package api - -import ( - "context" - - "github.com/keboola/keboola-sdk-go/v2/pkg/request" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -type wakeupBody struct { - DesiredState string `json:"desiredState"` -} - -func (a *API) WakeupApp(appID AppID) request.APIRequest[request.NoResult] { - return request.NewAPIRequest(request.NoResult{}, a.newRequest(). - WithError(&Error{}). - WithOnError(func(ctx context.Context, response request.HTTPResponse, err error) error { - span := trace.SpanFromContext(ctx) - attrs := []attribute.KeyValue{ - attribute.Int(attrSandboxesServiceStatusCode, response.StatusCode()), - } - span.SetAttributes(attrs...) - return err - }). - WithPatch("apps/{appId}"). - AndPathParam("appId", appID.String()). - WithJSONBody(wakeupBody{ - DesiredState: "running", - }), - ) -} diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go new file mode 100644 index 0000000000..2ec1d8d70b --- /dev/null +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/types.go @@ -0,0 +1,56 @@ +// Package k8sapp provides watching and patching of App CRDs in Kubernetes. +package k8sapp + +import ( + "net/url" + + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + Group = "apps.keboola.com" + Version = "v2" + Resource = "apps" +) + +// AppGVR is the GroupVersionResource for the App CRD. +var AppGVR = schema.GroupVersionResource{Group: Group, Version: Version, Resource: Resource} + +// AppActualState is the observed state of the app, read from .status.currentState. +type AppActualState string + +const ( + AppActualStateStopped AppActualState = "Stopped" + AppActualStateRunning AppActualState = "Running" + AppActualStateStarting AppActualState = "Starting" + AppActualStateStopping AppActualState = "Stopping" +) + +// appObject is a minimal struct for unmarshalling App CRD objects — only the fields we need. +type appObject struct { + Spec appSpec `json:"spec"` + Status appStatus `json:"status"` +} + +type appSpec struct { + AppID string `json:"appId"` + AutoRestartEnabled *bool `json:"autoRestartEnabled,omitempty"` +} + +// AppInfo is the cached state for an app, read from the K8s watcher. +type AppInfo struct { + ActualState AppActualState + AutoRestartEnabled bool + // UpstreamTarget is the pre-parsed URL from .status.appsProxy.upstreamUrl. + // Nil when the field is absent or unparseable. + UpstreamTarget *url.URL +} + +type appStatus struct { + CurrentState AppActualState `json:"currentState"` + AppsProxy appsProxy `json:"appsProxy,omitempty"` +} + +type appsProxy struct { + UpstreamURL string `json:"upstreamUrl,omitempty"` +} diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go new file mode 100644 index 0000000000..d556168788 --- /dev/null +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher.go @@ -0,0 +1,246 @@ +package k8sapp + +import ( + "context" + "encoding/json" + "net/url" + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + + "github.com/keboola/keboola-as-code/internal/pkg/log" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" +) + +// entry stores the K8s object name and last observed state for an app. +type entry struct { + k8sName string + state AppActualState + autoRestartEnabled bool + upstreamTarget *url.URL // pre-parsed; nil when appsProxy.upstreamUrl absent/invalid +} + +// StateWatcher watches App CRDs in Kubernetes and provides a local cache of app states. +type StateWatcher struct { + client dynamic.Interface + namespace string + logger log.Logger + hasSynced cache.InformerSynced + // byName: K8s object name → AppID + byName sync.Map + // byAppID: AppID → entry{k8sName, state} + byAppID sync.Map +} + +type dependencies interface { + Logger() log.Logger + Process() *servicectx.Process +} + +// NewDynamicClient creates a Kubernetes dynamic client from kubeconfig path or in-cluster config. +// If kubeconfigPath is empty, in-cluster config is used. +func NewDynamicClient(kubeconfigPath string) (dynamic.Interface, error) { + var cfg *rest.Config + var err error + if kubeconfigPath != "" { + cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath) + } else { + cfg, err = rest.InClusterConfig() + } + if err != nil { + return nil, err + } + return dynamic.NewForConfig(cfg) +} + +// NewStateWatcher creates and starts a StateWatcher that watches App CRDs in the given namespace. +// It registers the informer lifecycle with the process. +func NewStateWatcher(d dependencies, client dynamic.Interface, namespace string) *StateWatcher { + w := &StateWatcher{ + client: client, + namespace: namespace, + logger: d.Logger().WithComponent("k8sapp.watcher"), + hasSynced: func() bool { return false }, + } + + ctx, cancel := context.WithCancel(context.Background()) + d.Process().OnShutdown(func(context.Context) { + cancel() + }) + + lw := &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return client.Resource(AppGVR).Namespace(namespace).List(ctx, opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return client.Resource(AppGVR).Namespace(namespace).Watch(ctx, opts) + }, + } + + informer := cache.NewSharedIndexInformer( + lw, + &unstructured.Unstructured{}, + // No resync — rely on watch events only. + 0, + cache.Indexers{}, + ) + + _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + w.handleUpsert(ctx, obj) + }, + UpdateFunc: func(_, newObj any) { + w.handleUpsert(ctx, newObj) + }, + DeleteFunc: func(obj any) { + w.handleDelete(ctx, obj) + }, + }) + if err != nil { + w.logger.Errorf(ctx, "failed to add event handler to App informer: %s", err) + } + + w.hasSynced = informer.HasSynced + + go informer.Run(ctx.Done()) + + // Log when the cache has synced so operators know the watcher is ready. + go func() { + if cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) { + w.logger.Infof(ctx, "App CRD cache synced for namespace %q", namespace) + } + }() + + return w +} + +// GetState returns the cached AppInfo for the app. Returns (AppInfo{}, false) if not yet cached. +func (w *StateWatcher) GetState(appID api.AppID) (AppInfo, bool) { + v, ok := w.byAppID.Load(appID) + if !ok { + return AppInfo{}, false + } + e := v.(entry) + return AppInfo{ + ActualState: e.state, + AutoRestartEnabled: e.autoRestartEnabled, + UpstreamTarget: e.upstreamTarget, + }, true +} + +// WaitForCacheSync blocks until the informer cache has completed its initial list, +// the stopCh is closed, or the context is cancelled. +// Intended for use in tests to ensure the watch is established before creating objects. +func (w *StateWatcher) WaitForCacheSync(ctx context.Context) bool { + return cache.WaitForCacheSync(ctx.Done(), w.hasSynced) +} + +// SetDesiredRunning patches .spec.state = "Running" on the App CRD for the given appID. +// If the appID is not yet in the cache, no patch is sent. +func (w *StateWatcher) SetDesiredRunning(ctx context.Context, appID api.AppID) error { + v, ok := w.byAppID.Load(appID) + if !ok { + return nil + } + e := v.(entry) + + patch, err := json.Marshal(map[string]any{ + "spec": map[string]any{ + "state": "Running", + }, + }) + if err != nil { + return err + } + + _, err = w.client.Resource(AppGVR).Namespace(w.namespace).Patch( + ctx, + e.k8sName, + k8stypes.MergePatchType, + patch, + metav1.PatchOptions{}, + ) + return err +} + +func (w *StateWatcher) handleUpsert(ctx context.Context, obj any) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return + } + k8sName := u.GetName() + + data, err := u.MarshalJSON() + if err != nil { + w.logger.Errorf(ctx, "failed to marshal App CRD %q: %s", k8sName, err) + return + } + + var appObj appObject + if err = json.Unmarshal(data, &appObj); err != nil { + w.logger.Errorf(ctx, "failed to unmarshal App CRD %q: %s", k8sName, err) + return + } + + if appObj.Spec.AppID == "" { + w.logger.Warnf(ctx, "App CRD %q has empty spec.appId, skipping", k8sName) + return + } + + autoRestartEnabled := true + if appObj.Spec.AutoRestartEnabled != nil { + autoRestartEnabled = *appObj.Spec.AutoRestartEnabled + } + + var upstreamTarget *url.URL + if rawURL := appObj.Status.AppsProxy.UpstreamURL; rawURL != "" { + if t, err := url.Parse(rawURL); err == nil { + upstreamTarget = t + } else { + w.logger.Warnf(ctx, "App CRD %q (appID=%s) invalid upstream URL %q from appsProxy.upstreamUrl: %s", k8sName, appObj.Spec.AppID, rawURL, err) + } + } + + appID := api.AppID(appObj.Spec.AppID) + w.byName.Store(k8sName, appID) + w.byAppID.Store(appID, entry{ + k8sName: k8sName, + state: appObj.Status.CurrentState, + autoRestartEnabled: autoRestartEnabled, + upstreamTarget: upstreamTarget, + }) + w.logger.Debugf(ctx, "App CRD %q (appID=%s) state updated: actualState=%q autoRestartEnabled=%v upstreamTarget=%v", k8sName, appID, appObj.Status.CurrentState, autoRestartEnabled, upstreamTarget != nil) +} + +func (w *StateWatcher) handleDelete(ctx context.Context, obj any) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + // Handle tombstone objects from the cache. + tombstone, ok2 := obj.(cache.DeletedFinalStateUnknown) + if !ok2 { + return + } + u, ok = tombstone.Obj.(*unstructured.Unstructured) + if !ok { + return + } + } + k8sName := u.GetName() + + v, loaded := w.byName.LoadAndDelete(k8sName) + if !loaded { + return + } + appID := v.(api.AppID) + w.byAppID.Delete(appID) + w.logger.Debugf(ctx, "App CRD %q (appID=%s) removed from cache", k8sName, appID) +} diff --git a/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go new file mode 100644 index 0000000000..d19be19aa5 --- /dev/null +++ b/internal/pkg/service/appsproxy/dataapps/k8sapp/watcher_test.go @@ -0,0 +1,213 @@ +package k8sapp_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/keboola/keboola-as-code/internal/pkg/log" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" + "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" +) + +const testNamespace = "keboola" + +// watcherDeps implements k8sapp.StateWatcher dependencies for tests. +type watcherDeps struct { + logger log.Logger + proc *servicectx.Process +} + +func newTestDeps(t *testing.T) *watcherDeps { + t.Helper() + logger := log.NewNopLogger() + proc := servicectx.New(servicectx.WithLogger(logger), servicectx.WithoutSignals()) + t.Cleanup(func() { + proc.Shutdown(context.Background(), nil) + proc.WaitForShutdown() + }) + return &watcherDeps{logger: logger, proc: proc} +} + +func (d *watcherDeps) Logger() log.Logger { return d.logger } +func (d *watcherDeps) Process() *servicectx.Process { return d.proc } + +// newFakeClient creates a fake dynamic client with the App list kind registered. +func newFakeClient(objects ...runtime.Object) *k8sfake.FakeDynamicClient { + scheme := runtime.NewScheme() + return k8sfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + k8sapp.AppGVR: "AppList", + }, objects...) +} + +// newAppObject creates an unstructured App CRD object. +func newAppObject(k8sName, appID string, state k8sapp.AppActualState) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": k8sapp.Group + "/" + k8sapp.Version, + "kind": "App", + "metadata": map[string]any{ + "name": k8sName, + "namespace": testNamespace, + }, + "spec": map[string]any{ + "appId": appID, + }, + "status": map[string]any{ + "currentState": string(state), + }, + }, + } +} + +// newAppObjectWithUpstreamURL creates an unstructured App CRD object with appsProxy.upstreamUrl set. +func newAppObjectWithUpstreamURL(k8sName, appID string, state k8sapp.AppActualState, upstreamURL string) *unstructured.Unstructured { + obj := newAppObject(k8sName, appID, state) + obj.Object["status"].(map[string]any)["appsProxy"] = map[string]any{"upstreamUrl": upstreamURL} + return obj +} + +func TestStateWatcher_GetState_UnknownWhenEmpty(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + watcher := k8sapp.NewStateWatcher(newTestDeps(t), fakeClient, testNamespace) + + info, ok := watcher.GetState(api.AppID("app-123")) + assert.False(t, ok) + assert.Empty(t, info.ActualState) +} + +func TestStateWatcher_GetState_AfterCacheSync(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + // Add the App CRD object to the fake tracker before the watcher starts. + appObj := newAppObject("my-app-k8s", "app-123", k8sapp.AppActualStateStopped) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + assert.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("app-123")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) +} + +func TestStateWatcher_SetDesiredRunning(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + appObj := newAppObject("my-app-k8s", "app-123", k8sapp.AppActualStateStopped) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + // Wait for the informer to cache the object. + require.Eventually(t, func() bool { + _, ok := watcher.GetState(api.AppID("app-123")) + return ok + }, 5*time.Second, 50*time.Millisecond) + + // Clear prior actions (list/watch from informer startup). + fakeClient.ClearActions() + + err = watcher.SetDesiredRunning(t.Context(), api.AppID("app-123")) + require.NoError(t, err) + + // Verify that a merge-patch action targeting App CRDs was recorded. + actions := fakeClient.Actions() + require.Len(t, actions, 1) + + pa, ok := actions[0].(k8stesting.PatchAction) + require.True(t, ok, "expected a PatchAction") + assert.Equal(t, k8stypes.MergePatchType, pa.GetPatchType()) + assert.Contains(t, string(pa.GetPatch()), `"state":"Running"`) +} + +func TestStateWatcher_SetDesiredRunning_NoOpWhenUnknown(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + watcher := k8sapp.NewStateWatcher(newTestDeps(t), fakeClient, testNamespace) + + // App not in K8s cache — SetDesiredRunning should be a no-op. + err := watcher.SetDesiredRunning(t.Context(), api.AppID("app-unknown")) + require.NoError(t, err) + + for _, a := range fakeClient.Actions() { + assert.NotEqual(t, "patch", a.GetVerb(), "unexpected PATCH for unknown app") + } +} + +func TestStateWatcher_GetState_UpstreamTarget(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + appObj := newAppObjectWithUpstreamURL("my-app-k8s", "app-123", k8sapp.AppActualStateRunning, "http://my-svc.keboola.svc.cluster.local:8888") + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + var info k8sapp.AppInfo + assert.Eventually(t, func() bool { + var ok bool + info, ok = watcher.GetState(api.AppID("app-123")) + return ok && info.UpstreamTarget != nil + }, 5*time.Second, 50*time.Millisecond) + + require.NotNil(t, info.UpstreamTarget) + assert.Equal(t, "http", info.UpstreamTarget.Scheme) + assert.Equal(t, "my-svc.keboola.svc.cluster.local:8888", info.UpstreamTarget.Host) +} + +func TestStateWatcher_GetState_UpstreamTarget_AbsentWhenMissing(t *testing.T) { + t.Parallel() + + fakeClient := newFakeClient() + d := newTestDeps(t) + + // App CRD without appsProxy.upstreamUrl. + appObj := newAppObject("my-app-k8s", "app-123", k8sapp.AppActualStateRunning) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + t.Context(), appObj, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + watcher := k8sapp.NewStateWatcher(d, fakeClient, testNamespace) + + assert.Eventually(t, func() bool { + _, ok := watcher.GetState(api.AppID("app-123")) + return ok + }, 5*time.Second, 50*time.Millisecond) + + info, ok := watcher.GetState(api.AppID("app-123")) + require.True(t, ok) + assert.Nil(t, info.UpstreamTarget) +} diff --git a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go index 3ce6f057c4..1e4d182c05 100644 --- a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go +++ b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup.go @@ -11,21 +11,18 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/syncmap" - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" ) -// Interval sets how often the proxy sends wakeup request to sandboxes service. -// If the last notification for the app was less than this Interval ago then the notification is skipped. -const ( - Interval = time.Second - wakeupErrorToBeSkipped = `can't have desired state "running". Currently is in state: "stopping", desired state: "stopped"` -) +// Interval sets how often the proxy sends a wakeup request. +// If the last wakeup for the app was less than this Interval ago then the wakeup is skipped. +const Interval = time.Second type Manager struct { clock clockwork.Clock logger log.Logger - api *api.API + watcher *k8sapp.StateWatcher stateMap *syncmap.SyncMap[api.AppID, state] } @@ -37,14 +34,14 @@ type state struct { type dependencies interface { Clock() clockwork.Clock Logger() log.Logger - AppsAPI() *api.API + AppStateWatcher() *k8sapp.StateWatcher } func NewManager(d dependencies) *Manager { return &Manager{ - clock: d.Clock(), - logger: d.Logger(), - api: d.AppsAPI(), + clock: d.Clock(), + logger: d.Logger(), + watcher: d.AppStateWatcher(), stateMap: syncmap.New[api.AppID, state](func(api.AppID) *state { return &state{lock: &sync.Mutex{}} }), @@ -55,38 +52,22 @@ func (l *Manager) Wakeup(ctx context.Context, appID api.AppID) error { // Get cache item or init an empty item item := l.stateMap.GetOrInit(appID) - // Only one notification runs in parallel. + // Only one wakeup runs in parallel. // If there is an in-flight update, we are waiting for its results. item.lock.Lock() defer item.lock.Unlock() - // Return config from cache if still valid now := l.clock.Now() - if now.Before(item.nextRequestAfter) { - // Skip if a notification was sent less than Interval ago + // Skip if a wakeup was sent less than Interval ago return nil } // Update nextRequestAfter time item.nextRequestAfter = now.Add(Interval) - // Send the notification - _, err := l.api.WakeupApp(appID).Send(ctx) - - // Check if it's a restart disabled error via context code - var apiErr *api.Error - if errors.As(err, &apiErr) && apiErr.HasRestartDisabled() { - // This is expected for apps with restart disabled, don't log as error - l.logger.Infof(ctx, `app "%s" has restart disabled`, appID) - return err // Still return the error so it can be handled by the proxy - } - - // If it does not succeed but app is currently stopping do not log it as error, log only other errors - // Instead of implementing state machine as in sandboxes service, we want to skip valid state that the - // pod is deallocating, and we want to wait till pod is `stopped` and we can `start` the pod again. - if err != nil && err.Error() != wakeupErrorToBeSkipped { - l.logger.Errorf(ctx, `failed sending wakeup request to Sandboxes Service about for app "%s": %s`, appID, err.Error()) + if err := l.watcher.SetDesiredRunning(ctx, appID); err != nil { + l.logger.Errorf(ctx, `failed setting desired state "Running" for app "%s": %s`, appID, err) return err } return nil diff --git a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go index 869bf1954e..09bb118138 100644 --- a/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go +++ b/internal/pkg/service/appsproxy/dataapps/wakeup/wakeup_test.go @@ -2,25 +2,57 @@ package wakeup_test import ( "context" - "fmt" - "net/http" "sync" "testing" "time" - "github.com/jarcoal/httpmock" "github.com/jonboulle/clockwork" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/atomic" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8stesting "k8s.io/client-go/testing" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/wakeup" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dependencies" commonDeps "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" ) +const testNamespace = "keboola" + +func newTestApp(appID string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": k8sapp.Group + "/" + k8sapp.Version, + "kind": "App", + "metadata": map[string]any{ + "name": "app-k8s-" + appID, + "namespace": testNamespace, + }, + "spec": map[string]any{ + "appId": appID, + }, + "status": map[string]any{ + "currentState": string(k8sapp.AppActualStateStopped), + }, + }, + } +} + +func patchCount(actions []k8stesting.Action) int { + count := 0 + for _, a := range actions { + if a.GetVerb() == "patch" { + count++ + } + } + return count +} + func TestManager_Wakeup(t *testing.T) { t.Parallel() @@ -29,32 +61,44 @@ func TestManager_Wakeup(t *testing.T) { d, mock := dependencies.NewMockedServiceScope(t, ctx, config.New(), commonDeps.WithClock(clk)) appID := api.AppID("app") + fakeClient := mock.TestFakeK8sClient() - transport := mock.MockedHTTPTransport() - transport.RegisterResponder( - http.MethodPatch, - fmt.Sprintf("%s/apps/%s", mock.TestConfig().SandboxesAPI.URL, appID), - httpmock.NewStringResponder(http.StatusOK, ""), + // Register app in fake K8s so SetDesiredRunning has a target. + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + ctx, newTestApp(string(appID)), metav1.CreateOptions{}, ) + require.NoError(t, err) manager := d.WakeupManager() + watcher := d.AppStateWatcher() - // The first request is send to the API - err := manager.Wakeup(ctx, appID) + // Wait for watcher cache to sync. + require.Eventually(t, func() bool { + _, ok := watcher.GetState(appID) + return ok + }, 5*time.Second, 50*time.Millisecond) + + // Clear list/watch actions from informer startup. + fakeClient.ClearActions() + + // The first wakeup sends a K8s PATCH. + err = manager.Wakeup(ctx, appID) require.NoError(t, err) - assert.Equal(t, 1, transport.GetTotalCallCount()) + assert.Equal(t, 1, patchCount(fakeClient.Actions())) - // Request is skipped, the Interval was not exceeded + // Second call is skipped because the Interval was not exceeded. + fakeClient.ClearActions() clk.Advance(time.Millisecond) err = manager.Wakeup(ctx, appID) require.NoError(t, err) - assert.Equal(t, 1, transport.GetTotalCallCount()) + assert.Equal(t, 0, patchCount(fakeClient.Actions())) - // Exceed the Interval + // After exceeding the Interval, a new PATCH is sent. + fakeClient.ClearActions() clk.Advance(wakeup.Interval) err = manager.Wakeup(ctx, appID) require.NoError(t, err) - assert.Equal(t, 2, transport.GetTotalCallCount()) + assert.Equal(t, 1, patchCount(fakeClient.Actions())) } func TestManager_Wakeup_Race(t *testing.T) { @@ -65,35 +109,43 @@ func TestManager_Wakeup_Race(t *testing.T) { d, mock := dependencies.NewMockedServiceScope(t, ctx, config.New(), commonDeps.WithClock(clk)) appID := api.AppID("app") + fakeClient := mock.TestFakeK8sClient() - transport := mock.MockedHTTPTransport() - transport.RegisterResponder( - http.MethodPatch, - fmt.Sprintf("%s/apps/%s", mock.TestConfig().SandboxesAPI.URL, appID), - httpmock.NewStringResponder(http.StatusOK, ""), + // Register app in fake K8s. + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace(testNamespace).Create( + ctx, newTestApp(string(appID)), metav1.CreateOptions{}, ) + require.NoError(t, err) manager := d.WakeupManager() + watcher := d.AppStateWatcher() - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + // Wait for watcher cache to sync. + require.Eventually(t, func() bool { + _, ok := watcher.GetState(appID) + return ok + }, 5*time.Second, 50*time.Millisecond) + + // Clear list/watch actions from informer startup. + fakeClient.ClearActions() + + raceCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() wg := sync.WaitGroup{} counter := atomic.NewInt64(0) - // Load configuration 10x in parallel + // Call Wakeup 10x in parallel. for range 10 { wg.Go(func() { - err := manager.Wakeup(ctx, appID) - require.NoError(t, err) - + require.NoError(t, manager.Wakeup(raceCtx, appID)) counter.Add(1) }) } - // Wait for all requests wg.Wait() - // Check total goroutines/requests count + // All goroutines completed. assert.Equal(t, int64(10), counter.Load()) - assert.Equal(t, 1, transport.GetTotalCallCount()) + // Only one K8s PATCH is sent due to rate-limiting. + assert.Equal(t, 1, patchCount(fakeClient.Actions())) } diff --git a/internal/pkg/service/appsproxy/dependencies/dependencies.go b/internal/pkg/service/appsproxy/dependencies/dependencies.go index 60fba1542b..b3f6be28e3 100644 --- a/internal/pkg/service/appsproxy/dependencies/dependencies.go +++ b/internal/pkg/service/appsproxy/dependencies/dependencies.go @@ -20,11 +20,14 @@ import ( "net/http" "github.com/jonboulle/clockwork" + "k8s.io/client-go/dynamic" + k8sfake "k8s.io/client-go/dynamic/fake" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/appconfig" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/notify" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/wakeup" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler" @@ -32,7 +35,6 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/upstream" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/pagewriter" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/common/httpclient" "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" @@ -79,12 +81,13 @@ type ServiceScope interface { PageWriter() *pagewriter.Writer NotifyManager() *notify.Manager WakeupManager() *wakeup.Manager + AppStateWatcher() *k8sapp.StateWatcher } type Mocked interface { dependencies.Mocked TestConfig() config.Config - TestDNSServer() *dnsmock.Server + TestFakeK8sClient() *k8sfake.FakeDynamicClient } // serviceScope implements APIScope interface. @@ -100,12 +103,19 @@ type serviceScope struct { appConfigLoader appconfig.Loader notifyManager *notify.Manager wakeupManager *wakeup.Manager + appStateWatcher *k8sapp.StateWatcher } type parentScopes interface { dependencies.BaseScope } +// k8sClientProvider is optionally implemented by parentScopes to supply an already-constructed +// dynamic client (used in tests to inject a fake client without real kubeconfig). +type k8sClientProvider interface { + K8sDynamicClient() dynamic.Interface +} + type parentScopesImpl struct { dependencies.BaseScope } @@ -172,7 +182,7 @@ func newServiceScope(ctx context.Context, parentScp parentScopes, cfg config.Con d.parentScopes = parentScp d.config = cfg - d.upstreamTransport, err = transport.New(d, cfg.DNSServer) + d.upstreamTransport, err = transport.New(d) if err != nil { return nil, err } @@ -185,6 +195,18 @@ func newServiceScope(ctx context.Context, parentScp parentScopes, cfg config.Con d.appsAPI = api.New(d.HTTPClient(), cfg.SandboxesAPI.URL, cfg.SandboxesAPI.Token) d.appConfigLoader = appconfig.NewLoader(d) d.notifyManager = notify.NewManager(d) + + var k8sClient dynamic.Interface + if provider, ok := parentScp.(k8sClientProvider); ok { + k8sClient = provider.K8sDynamicClient() + } else { + k8sClient, err = k8sapp.NewDynamicClient(cfg.K8s.Kubeconfig) + if err != nil { + return nil, err + } + } + d.appStateWatcher = k8sapp.NewStateWatcher(d, k8sClient, cfg.K8s.AppsNamespace) + d.wakeupManager = wakeup.NewManager(d) d.authProxyManager = authproxy.NewManager(d) d.upstreamManager = upstream.NewManager(d) @@ -232,3 +254,7 @@ func (v *serviceScope) NotifyManager() *notify.Manager { func (v *serviceScope) WakeupManager() *wakeup.Manager { return v.wakeupManager } + +func (v *serviceScope) AppStateWatcher() *k8sapp.StateWatcher { + return v.appStateWatcher +} diff --git a/internal/pkg/service/appsproxy/dependencies/mocked.go b/internal/pkg/service/appsproxy/dependencies/mocked.go index 21edb20bd7..f66eea85d6 100644 --- a/internal/pkg/service/appsproxy/dependencies/mocked.go +++ b/internal/pkg/service/appsproxy/dependencies/mocked.go @@ -6,9 +6,13 @@ import ( "testing" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + k8sfake "k8s.io/client-go/dynamic/fake" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" ) @@ -16,16 +20,23 @@ import ( // mocked implements Mocked interface. type mocked struct { dependencies.Mocked - config config.Config - dnsServer *dnsmock.Server + config config.Config + fakeK8sClient *k8sfake.FakeDynamicClient } func (v *mocked) TestConfig() config.Config { return v.config } -func (v *mocked) TestDNSServer() *dnsmock.Server { - return v.dnsServer +// TestFakeK8sClient returns the fake Kubernetes dynamic client used by this mock. +// Tests can use it to pre-populate App CRD objects and inspect PATCH actions. +func (v *mocked) TestFakeK8sClient() *k8sfake.FakeDynamicClient { + return v.fakeK8sClient +} + +// K8sDynamicClient implements k8sClientProvider, supplying the fake client to newServiceScope. +func (v *mocked) K8sDynamicClient() dynamic.Interface { + return v.fakeK8sClient } func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config, opts ...dependencies.MockedOption) (ServiceScope, Mocked) { @@ -51,22 +62,20 @@ func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config if cfg.SandboxesAPI.Token == "" { cfg.SandboxesAPI.Token = "my-token" } - - var dnsServer *dnsmock.Server - if cfg.DNSServer == "" { - dnsServer = dnsmock.New(commonMock.MockedDNSPort()) - require.NoError(tb, dnsServer.Start()) - tb.Cleanup(func() { - _ = dnsServer.Shutdown() - }) - - cfg.DNSServer = dnsServer.Addr() + if cfg.K8s.AppsNamespace == "" { + cfg.K8s.AppsNamespace = "keboola" } + // Create fake K8s dynamic client. The App list kind is registered so the informer can list CRDs. + scheme := runtime.NewScheme() + fakeClient := k8sfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + k8sapp.AppGVR: "AppList", + }) + // Validate config require.NoError(tb, configmap.ValidateAndNormalize(&cfg)) - mock := &mocked{Mocked: commonMock, config: cfg, dnsServer: dnsServer} + mock := &mocked{Mocked: commonMock, config: cfg, fakeK8sClient: fakeClient} scope, err := newServiceScope(ctx, mock, cfg) require.NoError(tb, err) diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go b/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go index 014ca7b0a9..142277a603 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go @@ -99,7 +99,7 @@ func (pw *pageWriter) WriteErrorPage(w http.ResponseWriter, req *http.Request, o } func (pw *pageWriter) ProxyErrorHandler(w http.ResponseWriter, req *http.Request, err error) { - pw.pageWriter.ProxyErrorHandler(w, req, pw.app, nil, err) + pw.pageWriter.ProxyErrorHandler(w, req, pw.app, err) } func (pw *pageWriter) WriteRobotsTxt(w http.ResponseWriter, req *http.Request) { diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/manager.go b/internal/pkg/service/appsproxy/proxy/apphandler/manager.go index c72b5844ae..d72ca9daa5 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/manager.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/manager.go @@ -29,9 +29,10 @@ type Manager struct { } type appHandlerWrapper struct { - lock *sync.Mutex - handler http.Handler - cancel context.CancelCauseFunc + lock *sync.Mutex + handler http.Handler + cancel context.CancelCauseFunc + serviceRef string // appsProxy.upstreamUrl in effect when this handler was created } type dependencies interface { @@ -70,12 +71,14 @@ func (m *Manager) HandlerFor(ctx context.Context, result appconfig.AppConfigResu return m.newErrorHandler(ctx, api.AppConfig{ID: result.AppID}, result.Err) } - // Create a new handler, if needed - if wrapper.handler == nil || result.Modified { + // Create a new handler, if needed (config changed or appsProxy.upstreamUrl changed) + currentRef := m.upstreamManager.CurrentServiceRef(result.AppID) + if wrapper.handler == nil || result.Modified || wrapper.serviceRef != currentRef { if wrapper.cancel != nil { wrapper.cancel(errors.New("configuration changed")) } wrapper.handler, wrapper.cancel = m.newHandler(ctx, result.AppConfig) + wrapper.serviceRef = currentRef } return wrapper.handler diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go index a00f52065e..577c5ed807 100644 --- a/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go +++ b/internal/pkg/service/appsproxy/proxy/apphandler/upstream/upstream.go @@ -18,12 +18,12 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/appconfig" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/notify" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/wakeup" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/chain" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/pagewriter" "github.com/keboola/keboola-as-code/internal/pkg/service/common/ctxattr" - svcErrors "github.com/keboola/keboola-as-code/internal/pkg/service/common/errors" "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" @@ -45,18 +45,18 @@ type Manager struct { configLoader appconfig.Loader notify *notify.Manager wakeup *wakeup.Manager + stateWatcher *k8sapp.StateWatcher config config.Config } type AppUpstream struct { - manager *Manager - app api.AppConfig - target *url.URL - handler *chain.Chain - wsHandler *chain.Chain - cancelWs context.CancelCauseFunc - activeWsCount atomic.Int64 - restartDisabled atomic.Bool + manager *Manager + app api.AppConfig + target *url.URL // parsed from appsProxy.upstreamUrl at creation; nil when absent + handler *chain.Chain + wsHandler *chain.Chain + cancelWs context.CancelCauseFunc + activeWsCount atomic.Int64 } type dependencies interface { @@ -68,6 +68,7 @@ type dependencies interface { AppConfigLoader() appconfig.Loader NotifyManager() *notify.Manager WakeupManager() *wakeup.Manager + AppStateWatcher() *k8sapp.StateWatcher Config() config.Config } @@ -81,6 +82,7 @@ func NewManager(d dependencies) *Manager { configLoader: d.AppConfigLoader(), notify: d.NotifyManager(), wakeup: d.WakeupManager(), + stateWatcher: d.AppStateWatcher(), config: d.Config(), } @@ -96,16 +98,24 @@ func (m *Manager) Shutdown(ctx context.Context) { m.wg.Wait() } +// CurrentServiceRef returns the appsProxy.upstreamUrl string for appID from the K8s cache. +// Returns "" when the app is not cached or the field is absent/invalid. +func (m *Manager) CurrentServiceRef(appID api.AppID) string { + info, ok := m.stateWatcher.GetState(appID) + if !ok || info.UpstreamTarget == nil { + return "" + } + return info.UpstreamTarget.String() +} + func (m *Manager) NewUpstream(ctx context.Context, app api.AppConfig) (upstream *AppUpstream, err error) { _, span := m.telemetry.Tracer().Start(ctx, "keboola.go.apps-proxy.upstream.NewUpstream") defer span.End(&err) - // Parse target - target, err := url.Parse(app.UpstreamAppURL) - if err != nil { - return nil, svcErrors.NewServiceUnavailableError(errors.PrefixErrorf(err, - `unable to parse upstream url for app "%s"`, app.IdAndName(), - )) + // Resolve target URL at creation time; immutable after this point. + var target *url.URL + if info, ok := m.stateWatcher.GetState(app.ID); ok { + target = info.UpstreamTarget // pre-parsed by watcher on CRD event; may be nil } // Create reverse proxy @@ -132,6 +142,40 @@ func (m *Manager) NewUpstream(ctx context.Context, app api.AppConfig) (upstream } func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request) error { + ctx := req.Context() + + // K8s state pre-check: if we know the app is not running, handle it synchronously + // without attempting DNS/upstream. Falls through if state is unknown or Running. + appInfo, ok := u.manager.stateWatcher.GetState(u.app.ID) + if ok { + u.manager.logger.Debugf(ctx, "app %q state check: actualState=%q autoRestartEnabled=%v", u.app.ID, appInfo.ActualState, appInfo.AutoRestartEnabled) + } else { + u.manager.logger.Debugf(ctx, "app %q state check: not in cache, forwarding to upstream", u.app.ID) + } + if ok && appInfo.ActualState != k8sapp.AppActualStateRunning { + if appInfo.ActualState == k8sapp.AppActualStateStarting { + u.manager.logger.Debugf(ctx, "app %q is starting, serving spinner page", u.app.ID) + u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + return nil + } + if !appInfo.AutoRestartEnabled { + u.manager.logger.Debugf(ctx, "app %q is not running and restart is disabled, serving restart-disabled page", u.app.ID) + u.manager.pageWriter.WriteRestartDisabledPage(rw, req, u.app) + return nil + } + u.manager.logger.Debugf(ctx, "app %q is not running, triggering wakeup and serving spinner page", u.app.ID) + u.wakeup(ctx, errors.Errorf("app state is %s", appInfo.ActualState)) + u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + return nil + } + + // Target set at creation time; nil means appsProxy.upstreamUrl was absent. + if u.target == nil { + u.manager.logger.Debugf(ctx, "app %q has no appsProxy.upstreamUrl, serving spinner page", u.app.ID) + u.manager.pageWriter.WriteSpinnerPage(rw, req, u.app) + return nil + } + // Difference between regular and websocket request if strings.EqualFold(req.Header.Get("Connection"), "upgrade") && req.Header.Get("Upgrade") == "websocket" { return u.wsHandler.ServeHTTPOrError(rw, req) @@ -139,10 +183,25 @@ func (u *AppUpstream) ServeHTTPOrError(rw http.ResponseWriter, req *http.Request return u.handler.ServeHTTPOrError(rw, req) } -func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { +func (u *AppUpstream) newReverseProxy() *httputil.ReverseProxy { proxy := httputil.NewSingleHostReverseProxy(u.target) proxy.Transport = u.manager.transport - proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app, &u.restartDisabled) + proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app) + + // Clear req.Host so Go's HTTP client derives the Host header from req.URL.Host. + // httputil.NewSingleHostReverseProxy rewrites req.URL.Host but leaves req.Host + // (the actual Host header) set to the original incoming value, which causes + // upstreams that route by Host to return wrong responses. + origDirector := proxy.Director + proxy.Director = func(req *http.Request) { + origDirector(req) + req.Host = "" + } + return proxy +} + +func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { + proxy := u.newReverseProxy() return chain. New(chain.HandlerFunc(func(w http.ResponseWriter, req *http.Request) error { @@ -159,9 +218,7 @@ func (u *AppUpstream) newProxy(timeout time.Duration) *chain.Chain { } func (u *AppUpstream) newWebsocketProxy(timeout time.Duration) *chain.Chain { - proxy := httputil.NewSingleHostReverseProxy(u.target) - proxy.Transport = u.manager.transport - proxy.ErrorHandler = u.manager.pageWriter.ProxyErrorHandlerFor(u.app, &u.restartDisabled) + proxy := u.newReverseProxy() return chain. New(chain.HandlerFunc(func(w http.ResponseWriter, req *http.Request) error { @@ -194,13 +251,6 @@ func (u *AppUpstream) trace() chain.Middleware { GotConn: func(connInfo httptrace.GotConnInfo) { u.notify(ctx) }, - DNSDone: func(info httptrace.DNSDoneInfo) { - if info.Err != nil { - u.wakeup(ctx, info.Err) - } else { - u.restartDisabled.Store(false) - } - }, }) return next.ServeHTTPOrError(w, req.WithContext(reqCtx)) @@ -235,13 +285,6 @@ func (u *AppUpstream) wakeup(ctx context.Context, err error) { // Error is already logged by the Wakeup method itself. err := u.manager.wakeup.Wakeup(wakeupCtx, u.app.ID) //nolint:contextcheck - - // Check for restart disabled error - var apiErr *api.Error - if errors.As(err, &apiErr) && apiErr.HasRestartDisabled() { - u.restartDisabled.Store(true) - } - span.End(&err) }) } diff --git a/internal/pkg/service/appsproxy/proxy/pagewriter/error.go b/internal/pkg/service/appsproxy/proxy/pagewriter/error.go index 39a10a5137..27e603b9ff 100644 --- a/internal/pkg/service/appsproxy/proxy/pagewriter/error.go +++ b/internal/pkg/service/appsproxy/proxy/pagewriter/error.go @@ -4,7 +4,6 @@ import ( "net" "net/http" "strings" - "sync/atomic" "go.opentelemetry.io/otel/attribute" semconv "go.opentelemetry.io/otel/semconv/v1.17.0" @@ -27,20 +26,13 @@ type errorPageData struct { ExceptionID string } -func (pw *Writer) ProxyErrorHandlerFor(app api.AppConfig, restartDisabled *atomic.Bool) func(w http.ResponseWriter, req *http.Request, err error) { +func (pw *Writer) ProxyErrorHandlerFor(app api.AppConfig) func(w http.ResponseWriter, req *http.Request, err error) { return func(w http.ResponseWriter, req *http.Request, err error) { - pw.ProxyErrorHandler(w, req, app, restartDisabled, err) + pw.ProxyErrorHandler(w, req, app, err) } } -func (pw *Writer) ProxyErrorHandler(w http.ResponseWriter, req *http.Request, app api.AppConfig, restartDisabled *atomic.Bool, err error) { - // Check for restart disabled error - if restartDisabled.Load() { - pw.logger.Info(req.Context(), "app has restart disabled, rendering restart disabled page") - pw.WriteRestartDisabledPage(w, req, app) - return - } - +func (pw *Writer) ProxyErrorHandler(w http.ResponseWriter, req *http.Request, app api.AppConfig, err error) { var dnsError *net.DNSError if errors.As(err, &dnsError) { pw.logger.Info(req.Context(), "app is not running, rendering spinner page") diff --git a/internal/pkg/service/appsproxy/proxy/proxy_test.go b/internal/pkg/service/appsproxy/proxy/proxy_test.go index 44ac508a80..eca265c6cc 100644 --- a/internal/pkg/service/appsproxy/proxy/proxy_test.go +++ b/internal/pkg/service/appsproxy/proxy/proxy_test.go @@ -23,7 +23,6 @@ import ( "github.com/coder/websocket" "github.com/coder/websocket/wsjson" "github.com/keboola/go-utils/pkg/wildcards" - "github.com/miekg/dns" "github.com/oauth2-proxy/mockoidc" "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/logger" "github.com/stretchr/testify/assert" @@ -36,16 +35,21 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8stypes "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/dynamic/fake" + "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/config" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/auth/provider" + "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/k8sapp" proxyDependencies "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/logging" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/pagewriter" "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/testutil" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" @@ -54,9 +58,9 @@ import ( type testCase struct { name string - run func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) + setupK8s func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) + run func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) expectedNotifications map[string]int - expectedWakeUps map[string]int expectedSpans tracetest.SpanStubs } @@ -66,7 +70,7 @@ func TestAppProxyRouter(t *testing.T) { testCases := []testCase{ { name: "health-check", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to health-check endpoint request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://hub.keboola.local/health-check", nil) require.NoError(t, err) @@ -78,11 +82,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, "OK\n", string(body)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "missing-app-id", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request without app id request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://hub.keboola.local/", nil) require.NoError(t, err) @@ -94,11 +97,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Unexpected domain, missing application ID.") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "unknown-app-id", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to unknown app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://unknown.hub.keboola.local/health-check", nil) require.NoError(t, err) @@ -110,11 +112,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), html.EscapeString(`Application "unknown" not found in the stack.`)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "broken-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to broken app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://broken.hub.keboola.local/", nil) require.NoError(t, err) @@ -127,7 +128,6 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, expectedSpans: []tracetest.SpanStub{ { Name: "keboola.go.common.dependencies.NewBaseScope", @@ -219,7 +219,7 @@ func TestAppProxyRouter(t *testing.T) { }, { name: "no-rule-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with no path rules request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://norule.hub.keboola.local/", nil) require.NoError(t, err) @@ -232,11 +232,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "wrong-rule-type-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid1.hub.keboola.local/", nil) require.NoError(t, err) @@ -249,11 +248,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "missing-referenced-provider", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid2.hub.keboola.local/", nil) require.NoError(t, err) @@ -266,11 +264,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "empty-allowed-roles-array-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid3.hub.keboola.local/", nil) require.NoError(t, err) @@ -283,11 +280,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "unknown-provider-app", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with unknown provider request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid4.hub.keboola.local/", nil) require.NoError(t, err) @@ -300,11 +296,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "not-empty-providers-auth-required-false", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to app with invalid rule type request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://invalid5.hub.keboola.local/", nil) require.NoError(t, err) @@ -317,11 +312,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "redirect-to-canonical-host", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Redirect to the canonical URL (match cookies domain) request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://foo-bar-123.hub.keboola.local/some/data/app/url?foo=bar", nil) require.NoError(t, err) @@ -332,11 +326,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, "https://public-123.hub.keboola.local/some/data/app/url?foo=bar", location) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "redirect-to-host-lowercase", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://foo-12345.hub.keboola.local/some/data/app/url?foo=bar", nil) require.NoError(t, err) response, err := client.Do(request) @@ -346,11 +339,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, "https://lowercase-12345.hub.keboola.local/some/data/app/url?foo=bar", location) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() // Request to public app @@ -364,11 +356,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), `Request to application failed.`) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-sub-url", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/some/data/app/url?foo=bar", nil) request.Header.Set("User-Agent", "Internet Exploder") @@ -395,11 +386,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "private-app-verified-email", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(true), @@ -482,11 +472,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "private-app-unauthorized", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to private app (unauthorized) request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) @@ -517,11 +506,10 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-missing-csrf-token", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -588,11 +576,10 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-group-mismatch", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "manager@keboola.com", Groups: []string{"manager"}, @@ -643,11 +630,10 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-unverified-email", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(false), @@ -696,11 +682,10 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-oidc-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to private app (unauthorized) request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) @@ -748,11 +733,10 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusFound, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() m[0].QueueUser(&mockoidc.MockUser{ @@ -840,11 +824,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), `Request to application failed.`) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-basic-flow", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin", "manager"}, @@ -946,11 +929,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "multi": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-redirect-to-selection-page", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -978,11 +960,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), htmlLinkTo(`https://multi.hub.keboola.local/_proxy/selection?provider=oidc2`)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-unverified-email", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(false), @@ -1047,11 +1028,10 @@ func TestAppProxyRouter(t *testing.T) { require.Equal(t, http.StatusUnauthorized, response.StatusCode) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-down", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() m[1].QueueUser(&mockoidc.MockUser{ @@ -1125,11 +1105,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), `Request to application failed.`) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-broken-provider", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { appServer.Close() // Provider selection @@ -1153,11 +1132,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), pagewriter.ExceptionIDPrefix) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-websocket", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), time.Minute) defer cancel() @@ -1181,11 +1159,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "private-app-websocket-unauthorized", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), time.Minute) defer cancel() @@ -1200,11 +1177,10 @@ func TestAppProxyRouter(t *testing.T) { require.Contains(t, err.Error(), "failed to WebSocket dial: expected handshake response status code 101 but got 302") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-websocket", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", EmailVerified: new(true), @@ -1273,11 +1249,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "websocket-connection-check", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() @@ -1304,11 +1279,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "111": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "multi-app-websocket", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1397,11 +1371,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "multi": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "prefix-app-no-auth", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public part of the app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://prefix.hub.keboola.local/public", nil) require.NoError(t, err) @@ -1433,11 +1406,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "prefix-app-api-auth", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1496,11 +1468,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "prefix-app-web-auth", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1578,11 +1549,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "shared-provider", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[1].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1666,11 +1636,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "prefix": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "configuration-change", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) @@ -1713,11 +1682,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "concurrency-test", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() @@ -1750,21 +1718,38 @@ func TestAppProxyRouter(t *testing.T) { assert.Equal(t, int64(100), counter.Load()) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-app-wakeup", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - - // Request to public app - fails because the app doesn't have a DNS record + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + // Request to public app - fails because the app is Stopped request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) response, err := client.Do(request) require.NoError(t, err) require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - dnsServer.AddARecord(dns.Fqdn("app.local"), net.ParseIP("127.0.0.1")) + // Patch app back to Running + patch := []byte(`{"status":{"currentState":"Running"}}`) + _, err = fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && info.ActualState == k8sapp.AppActualStateRunning + }, 5*time.Second, 50*time.Millisecond) // Request to public app request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) @@ -1779,160 +1764,39 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{ - "123": 1, - }, }, { name: "public-app-wakeup-only", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - - // Request to public app - fails because the app doesn't have a DNS record - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - body, err := io.ReadAll(response.Body) + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) require.NoError(t, err) - assert.Contains(t, string(body), "Starting your application...") - - // Expect wakeup but no notification since there was an authorized request to the app but not while it was running. + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) }, - expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{ - "123": 1, - }, - }, - { - name: "restart-disabled", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - - service.WakeUpOverrides["123"] = func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusBadRequest) - _, _ = fmt.Fprintln(w, `{ - "code": 0, - "context": { - "code": "apps.restartDisabled" - }, - "error": "App restart is disabled. Contact app maintainer.", - "exceptionId": "exception-208db995c92ed365d47bcc701ae4d802", - "status": "error" -}`) - } - // Request to public app - fails because the app doesn't have a DNS record + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + // Request to public app - fails because the app is Stopped, triggers wakeup request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) response, err := client.Do(request) require.NoError(t, err) - // First request returns 503 Service Unavailable (Spinner) because wakeup is async require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) body, err := io.ReadAll(response.Body) require.NoError(t, err) assert.Contains(t, string(body), "Starting your application...") - // Wait for async wakeup to complete - assert.Eventually(t, func() bool { - // Second request should return 404 Not Found (Restart Disabled) - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - if response.StatusCode == http.StatusServiceUnavailable { - body, err := io.ReadAll(response.Body) - require.NoError(t, err) - return strings.Contains(string(body), "Application Disabled") - } - return false - }, 5*time.Second, 100*time.Millisecond) - // Expect wakeup but no notification since there was an authorized request to the app but not while it was running. }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, - }, - { - name: "restart-disabled-flag-reset-on-dns-success", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - // Phase 1: Set restart disabled state - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - - service.WakeUpOverrides["123"] = func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusBadRequest) - _, _ = fmt.Fprintln(w, `{ - "code": 0, - "context": { - "code": "apps.restartDisabled" - }, - "error": "App restart is disabled. Contact app maintainer.", - "exceptionId": "exception-208db995c92ed365d47bcc701ae4d802", - "status": "error" -}`) - } - - // Request to public app - fails because the app doesn't have a DNS record - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - // First request returns 503 Service Unavailable (Spinner) because wakeup is async - require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - - // Wait for async wakeup to complete and flag to be set - assert.Eventually(t, func() bool { - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - body, err := io.ReadAll(response.Body) - require.NoError(t, err) - return response.StatusCode == http.StatusServiceUnavailable && strings.Contains(string(body), "Application Disabled") - }, 5*time.Second, 100*time.Millisecond) - - // Phase 2: Simulate app becoming available (restart enabled) - // Add DNS record back and remove wakeup override - dnsServer.AddARecord(dns.Fqdn("app.local"), net.ParseIP("127.0.0.1")) - - // Wait for DNS to be fully propagated before removing override - assert.Eventually(t, func() bool { - request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err := client.Do(request) - require.NoError(t, err) - _, _ = io.ReadAll(response.Body) - return response.StatusCode == http.StatusOK - }, 5*time.Second, 100*time.Millisecond) - - // Now safe to remove override - delete(service.WakeUpOverrides, "123") - - // Request should now succeed - DNS resolution succeeds, flag gets reset - request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err = client.Do(request) - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - body, err := io.ReadAll(response.Body) - require.NoError(t, err) - assert.Contains(t, string(body), "Hello, client") - - // Verify flag stays reset - subsequent requests should continue to work - request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) - require.NoError(t, err) - response, err = client.Do(request) - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - }, - expectedNotifications: map[string]int{"123": 1}, - expectedWakeUps: map[string]int{}, }, + { name: "private-one-provider-selector", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request provider selector page - no auth provider request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/_proxy/selection", nil) require.NoError(t, err) @@ -1944,13 +1808,21 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), htmlLinkTo(`https://oidc.hub.keboola.local/_proxy/selection?provider=oidc`)) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "private-app-wakeup", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-oidc", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("oidc")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -1997,14 +1869,23 @@ func TestAppProxyRouter(t *testing.T) { require.NoError(t, err) require.Equal(t, http.StatusFound, response.StatusCode) - // Request to private app (authorized but missing dns, triggers wakeup) + // Request to private app (authorized but app is Stopped, triggers wakeup) request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) response, err = client.Do(request) require.NoError(t, err) require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - dnsServer.AddARecord(dns.Fqdn("app.local"), net.ParseIP("127.0.0.1")) + // Patch app back to Running + patch := []byte(`{"status":{"currentState":"Running"}}`) + _, err = fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-oidc", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("oidc")) + return ok && info.ActualState == k8sapp.AppActualStateRunning + }, 5*time.Second, 50*time.Millisecond) // Request to private app (authorized) request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) @@ -2016,15 +1897,21 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{ - "oidc": 1, - }, }, { name: "private-app-wakeup-only", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + patch := []byte(`{"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-oidc", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("oidc")) + return ok && info.ActualState == k8sapp.AppActualStateStopped + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -2071,7 +1958,7 @@ func TestAppProxyRouter(t *testing.T) { require.NoError(t, err) require.Equal(t, http.StatusFound, response.StatusCode) - // Request to private app (authorized but missing dns, triggers wakeup) + // Request to private app (authorized but app is Stopped, triggers wakeup) request, err = http.NewRequestWithContext(t.Context(), http.MethodGet, "https://oidc.hub.keboola.local/", nil) require.NoError(t, err) response, err = client.Do(request) @@ -2084,15 +1971,10 @@ func TestAppProxyRouter(t *testing.T) { // Expect wakeup but no notification since there was an authorized request to the app but not while it was running. }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{ - "oidc": 1, - }, }, { name: "private-app-no-wakeup", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { - dnsServer.RemoveARecords(dns.Fqdn("app.local")) - + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -2142,11 +2024,10 @@ func TestAppProxyRouter(t *testing.T) { // Expect no notification or wakeup because there was never an authorized request to the app }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-wrong-login-no-password", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/", nil) require.NoError(t, err) @@ -2169,11 +2050,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Please enter a correct password.") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-wrong-login", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/", nil) require.NoError(t, err) @@ -2196,11 +2076,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Please enter a correct password.") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-correct-app-url", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/app/url", nil) require.NoError(t, err) @@ -2244,11 +2123,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "auth": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-correct-login", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request public basic auth app request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://basic-auth.hub.keboola.local/_proxy/form", nil) require.NoError(t, err) @@ -2295,11 +2173,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "auth": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-wrong-cookie", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Access with cookie request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://basic-auth.hub.keboola.local/", nil) request.AddCookie(&http.Cookie{Name: "proxyBasicAuth", Value: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015a"}) @@ -2312,11 +2189,10 @@ func TestAppProxyRouter(t *testing.T) { assert.Contains(t, string(body), "Cookie has expired") }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-cookie", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Access with cookie request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://basic-auth.hub.keboola.local/", nil) request.AddCookie(&http.Cookie{Name: "proxyBasicAuth", Value: "$2a$10$65mF6LI2F0Nm9PkQk8DlJu.C5jD.fseeXWn9CCGmDxLPomikYWtte"}) @@ -2332,11 +2208,10 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "auth": 1, }, - expectedWakeUps: map[string]int{}, }, { name: "public-basic-auth-sign-out", - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Access with cookie request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://basic-auth.hub.keboola.local/_proxy/sign_out", nil) request.AddCookie(&http.Cookie{Name: "proxyBasicAuth", Value: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"}) @@ -2358,14 +2233,42 @@ func TestAppProxyRouter(t *testing.T) { require.Empty(t, response.Cookies()) }, expectedNotifications: map[string]int{}, - expectedWakeUps: map[string]int{}, + }, + { + name: "restart-disabled", + setupK8s: func(t *testing.T, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + // The default setup already created app "123" as Running. + // Patch it to Stopped + autoRestartEnabled=false. + patch := []byte(`{"spec":{"autoRestartEnabled":false},"status":{"currentState":"Stopped"}}`) + _, err := fakeClient.Resource(k8sapp.AppGVR).Namespace("keboola").Patch( + t.Context(), "app-123", k8stypes.MergePatchType, patch, metav1.PatchOptions{}, + ) + require.NoError(t, err) + + require.Eventually(t, func() bool { + info, ok := watcher.GetState(api.AppID("123")) + return ok && !info.AutoRestartEnabled + }, 5*time.Second, 50*time.Millisecond) + }, + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { + // Use canonical slug-based URL to bypass the slug redirect. + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://public-123.hub.keboola.local/", nil) + require.NoError(t, err) + response, err := client.Do(request) + require.NoError(t, err) + require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + assert.Contains(t, string(body), "Application Disabled") + }, + expectedNotifications: map[string]int{}, }, } publicAppTestCaseFactory := func(method string) testCase { return testCase{ name: "public-app-" + method, - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { // Request to public app request, err := http.NewRequestWithContext(t.Context(), method, "https://public-123.hub.keboola.local/", nil) require.NoError(t, err) @@ -2379,7 +2282,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "123": 1, }, - expectedWakeUps: map[string]int{}, } } @@ -2395,7 +2297,7 @@ func TestAppProxyRouter(t *testing.T) { privateAppTestCaseFactory := func(method string) testCase { return testCase{ name: "private-app-oidc-" + method, - run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) { + run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, fakeClient *k8sfake.FakeDynamicClient, watcher *k8sapp.StateWatcher) { m[0].QueueUser(&mockoidc.MockUser{ Email: "admin@keboola.com", Groups: []string{"admin"}, @@ -2485,7 +2387,6 @@ func TestAppProxyRouter(t *testing.T) { expectedNotifications: map[string]int{ "oidc": 1, }, - expectedWakeUps: map[string]int{}, } } @@ -2526,14 +2427,13 @@ func TestAppProxyRouter(t *testing.T) { mocked.TestTelemetry().AssertSpans(t, tc.expectedSpans, opts...) } - dnsServer := mocked.TestDNSServer() - // Create test OIDC providers providers := testAuthProviders(t, pm) // Register data apps - appURL := testutil.AddAppDNSRecord(t, appServer, dnsServer) - appsAPI.Register(testDataApps(appURL, providers)) + appURL := testutil.AppServerURL(t, appServer) + apps := testDataApps(appURL, providers) + appsAPI.Register(apps) // Create proxy handler handler := createProxyHandler(ctx, d) @@ -2561,13 +2461,18 @@ func TestAppProxyRouter(t *testing.T) { client := createHTTPClient(t, proxyURL) - tc.run(t, client, providers, appServer, appsAPI, dnsServer) + // Default K8s setup: register all test apps as Running with appsProxy.upstreamUrl. + registerDefaultK8sApps(t, apps, mocked.TestFakeK8sClient(), d.AppStateWatcher(), appURL.String()) + if tc.setupK8s != nil { + tc.setupK8s(t, mocked.TestFakeK8sClient(), d.AppStateWatcher()) + } + + tc.run(t, client, providers, appServer, appsAPI, mocked.TestFakeK8sClient(), d.AppStateWatcher()) d.Process().Shutdown(t.Context(), errors.New("bye bye")) d.Process().WaitForShutdown() assert.Equal(t, tc.expectedNotifications, appsAPI.Notifications) - assert.Equal(t, tc.expectedWakeUps, appsAPI.WakeUps) assert.Empty(t, mocked.DebugLogger().ErrorMessages()) }) } @@ -2903,8 +2808,7 @@ func createDependencies(t *testing.T, ctx context.Context, sandboxesAPIURL strin cfg.CookieSecretSalt = string(secret) cfg.CsrfTokenSalt = string(csrfSecret) cfg.SandboxesAPI.URL = sandboxesAPIURL - port := pm.GetFreePort() - return proxyDependencies.NewMockedServiceScope(t, ctx, cfg, dependencies.WithRealHTTPClient(), dependencies.WithMockedDNSPort(port)) + return proxyDependencies.NewMockedServiceScope(t, ctx, cfg, dependencies.WithRealHTTPClient()) } func createProxyHandler(ctx context.Context, d proxyDependencies.ServiceScope) http.Handler { @@ -2962,3 +2866,48 @@ func extractMetaRefreshTag(t *testing.T, body []byte) string { func htmlLinkTo(url string) string { return fmt.Sprintf(` 0 { - // nolint: gosec // we don't need to use crypto.rand here - ip := resp.Answer[rand.Intn(len(resp.Answer))].(*dns.AAAA).AAAA.String() - return ip, nil - } - - return "", &net.DNSError{ - Err: fmt.Sprintf(`host not found: %s`, host), - Name: host, - Server: c.dnsServer, - IsNotFound: true, - } - } - - // nolint: gosec // we don't need to use crypto.rand here - ip := resp.Answer[rand.Intn(len(resp.Answer))].(*dns.A).A.String() - return ip, nil -} diff --git a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock.go b/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock.go deleted file mode 100644 index 2f9a58d27e..0000000000 --- a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock.go +++ /dev/null @@ -1,186 +0,0 @@ -// Based on https://github.com/kuritka/go-fake-dns -// License: MIT - -package dnsmock - -import ( - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/miekg/dns" - - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" -) - -type DNSRecordError struct { - Err error -} - -func (d DNSRecordError) Error() string { - return d.Err.Error() -} - -// Server acts as DNS server but returns mock values. -type Server struct { - server *dns.Server - records map[uint16][]dns.RR - updateLock *sync.Mutex -} - -func New(port int) *Server { - server := &Server{ - records: make(map[uint16][]dns.RR), - updateLock: &sync.Mutex{}, - } - - var handler dns.HandlerFunc = server.handleRequest - - server.server = &dns.Server{ - Addr: "[::]:" + strconv.FormatInt(int64(port), 10), - Net: "udp", - Handler: handler, - } - - return server -} - -func (s *Server) Start() error { - startedCh := make(chan struct{}) - s.server.NotifyStartedFunc = func() { - close(startedCh) - } - - serverErrCh := make(chan error, 1) - go func() { - serverErrCh <- s.server.ListenAndServe() - }() - - // Wait for DNS server startup - select { - case <-startedCh: - // ok - case err := <-serverErrCh: - return err - case <-time.After(5 * time.Second): - return errors.New("DNS server start timeout") - } - - return nil -} - -func (s *Server) Addr() string { - return s.server.PacketConn.LocalAddr().String() -} - -func (s *Server) Shutdown() error { - return s.server.Shutdown() -} - -func (s *Server) AddTXTRecord(fqdn string, strings ...string) { - t := &dns.TXT{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: 0}, - Txt: strings, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeTXT] = append(s.records[dns.TypeTXT], t) -} - -func (s *Server) AddNSRecord(fqdn, nsName string) { - ns := &dns.NS{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeNS, Class: dns.ClassINET, Ttl: 0}, - Ns: nsName, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeNS] = append(s.records[dns.TypeNS], ns) -} - -func (s *Server) AddAOrAAAARecord(fqdn string, ip net.IP) error { - ipv4 := ip.To4() - if ipv4 != nil { - return s.AddARecord(fqdn, ipv4) - } - - return s.AddAAAARecord(fqdn, ip) -} - -func (s *Server) AddARecord(fqdn string, ip net.IP) error { - ipv4 := ip.To4() - if ipv4 == nil { - return &DNSRecordError{Err: errors.New("Unable to create A record")} - } - - rr := &dns.A{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0}, - A: ipv4, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeA] = append(s.records[dns.TypeA], rr) - return nil -} - -func (s *Server) AddAAAARecord(fqdn string, ip net.IP) error { - ipv6 := ip.To16() - if ipv6 == nil { - return &DNSRecordError{Err: errors.New("Unable to create AAAA record")} - } - - rr := &dns.AAAA{ - Hdr: dns.RR_Header{Name: fqdn, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 0}, - AAAA: ipv6, - } - s.updateLock.Lock() - defer s.updateLock.Unlock() - s.records[dns.TypeAAAA] = append(s.records[dns.TypeAAAA], rr) - return nil -} - -func (s *Server) RemoveTXTRecords(fqdn string) { - s.removeRecords(dns.TypeTXT, fqdn) -} - -func (s *Server) RemoveNSRecords(fqdn string) { - s.removeRecords(dns.TypeNS, fqdn) -} - -func (s *Server) RemoveARecords(fqdn string) { - s.removeRecords(dns.TypeA, fqdn) -} - -func (s *Server) RemoveAAAARecords(fqdn string) { - s.removeRecords(dns.TypeAAAA, fqdn) -} - -func (s *Server) removeRecords(dnsType uint16, fqdn string) { - s.updateLock.Lock() - defer s.updateLock.Unlock() - records := []dns.RR{} - for _, rr := range s.records[dnsType] { - if rr.Header().Name != fqdn { - records = append(records, rr) - } - } - s.records[dnsType] = records -} - -func (s *Server) handleRequest(w dns.ResponseWriter, r *dns.Msg) { - msg := new(dns.Msg) - msg.SetReply(r) - msg.Compress = false - s.updateLock.Lock() - defer s.updateLock.Unlock() - if s.records[r.Question[0].Qtype] != nil { - for _, rr := range s.records[r.Question[0].Qtype] { - fqdn := strings.Split(rr.String(), "\t")[0] - if fqdn == r.Question[0].Name { - msg.Answer = append(msg.Answer, rr) - } - } - } - _ = w.WriteMsg(msg) -} diff --git a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock_test.go b/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock_test.go deleted file mode 100644 index f5a88db812..0000000000 --- a/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock/dnsmock_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Based on https://github.com/kuritka/go-fake-dns -// License: MIT - -package dnsmock_test - -import ( - "net" - "testing" - - "github.com/miekg/dns" - "github.com/stretchr/testify/require" - - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock" -) - -func TestMultipleTXTRecords(t *testing.T) { - t.Parallel() - - dnsMock := dnsmock.New(0) - dnsMock.AddTXTRecord("heartbeat-us.cloud.example.com.", "1") - dnsMock.AddTXTRecord("heartbeat-uk.cloud.example.com.", "2") - dnsMock.AddTXTRecord("heartbeat-eu.cloud.example.com.", "0", "6", "8") - err := dnsMock.Start() - require.NoError(t, err) - - defer dnsMock.Shutdown() - - g := new(dns.Msg) - g.SetQuestion("ip.blah.cloud.example.com.", dns.TypeTXT) - a, err := dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) - - g = new(dns.Msg) - g.SetQuestion("heartbeat-uk.cloud.example.com.", dns.TypeTXT) - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Len(t, a.Answer, 1) - require.Len(t, a.Answer[0].(*dns.TXT).Txt, 1) - require.Equal(t, "2", a.Answer[0].(*dns.TXT).Txt[0]) - - g = new(dns.Msg) - g.SetQuestion("heartbeat-eu.cloud.example.com.", dns.TypeTXT) - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Len(t, a.Answer, 1) - require.Len(t, a.Answer[0].(*dns.TXT).Txt, 3) - require.Equal(t, "0", a.Answer[0].(*dns.TXT).Txt[0]) - require.Equal(t, "6", a.Answer[0].(*dns.TXT).Txt[1]) - require.Equal(t, "8", a.Answer[0].(*dns.TXT).Txt[2]) -} - -func TestSimple(t *testing.T) { - t.Parallel() - - dnsMock := dnsmock.New(0) - dnsMock.AddNSRecord("blah.cloud.example.com.", "gslb-ns-us-cloud.example.com.") - dnsMock.AddNSRecord("blah.cloud.example.com.", "gslb-ns-uk-cloud.example.com.") - dnsMock.AddNSRecord("blah.cloud.example.com.", "gslb-ns-eu-cloud.example.com.") - dnsMock.AddTXTRecord("First", "Second", "Banana") - dnsMock.AddTXTRecord("White", "Red", "Purple") - dnsMock.AddARecord("ip.blah.cloud.example.com.", net.IPv4(10, 0, 1, 5)) - err := dnsMock.Start() - require.NoError(t, err) - - defer dnsMock.Shutdown() - - g := new(dns.Msg) - g.SetQuestion("ip.blah.cloud.example.com.", dns.TypeA) - a, err := dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.NotEmpty(t, a.Answer) - - dnsMock.RemoveARecords("ip.blah.cloud.example.com.") - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) -} - -func TestWrongARecord(t *testing.T) { - t.Parallel() - - dnsMock := dnsmock.New(0) - require.Error(t, dnsMock.AddARecord("ipv4.net.", net.IPv6loopback)) - require.Error(t, dnsMock.AddAAAARecord("ipv6.net.", net.IP{0, 0})) - err := dnsMock.Start() - require.NoError(t, err) - - defer dnsMock.Shutdown() - - g := new(dns.Msg) - g.SetQuestion("ipv4.net.", dns.TypeA) - a, err := dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) - - g.SetQuestion("ipv6.net.", dns.TypeAAAA) - a, err = dns.Exchange(g, dnsMock.Addr()) - require.NoError(t, err) - require.Empty(t, a.Answer) -} diff --git a/internal/pkg/service/appsproxy/proxy/transport/transport.go b/internal/pkg/service/appsproxy/proxy/transport/transport.go index d3222f4206..d290688744 100644 --- a/internal/pkg/service/appsproxy/proxy/transport/transport.go +++ b/internal/pkg/service/appsproxy/proxy/transport/transport.go @@ -1,21 +1,14 @@ -// Package transport provides HTTP transport for the reverse HTTP proxy with custom DNS resolving. -// DNS client is used to fast and reliable determine if the target data app is running. +// Package transport provides HTTP transport for the reverse HTTP proxy. package transport import ( - "context" - "net" "net/http" - "net/http/httptrace" "time" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/otel/attribute" "golang.org/x/net/http2" - "github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" ) // TLSHandshakeTimeout specifies the default timeout of TLS handshake. @@ -39,96 +32,17 @@ const HTTP2PingTimeout = 2 * time.Second // HTTP2WriteByteTimeout is the timeout after which the connection will be closed no data can be written to it. const HTTP2WriteByteTimeout = 15 * time.Second -const DNSResolveTimeout = 5 * time.Second - type dependencies interface { Telemetry() telemetry.Telemetry } // New creates new http transport intended to be used with NewSingleHostReverseProxy. -// DNS server address is loaded from the "/etc/resolv.conf". -func New(d dependencies, dnsServer string) (http.RoundTripper, error) { - return NewWithDNSServer(d, dnsServer) -} - -// NewWithDNSServer creates new http transport intended to be used with NewSingleHostReverseProxy. -func NewWithDNSServer(d dependencies, dnsServerAddress string) (http.RoundTripper, error) { +func New(d dependencies) (http.RoundTripper, error) { dialer := newDialer() - var dnsClient *dns.Client - var err error - if dnsServerAddress == "" { - dnsClient, err = dns.NewClientFromEtc(dialer) - if err != nil { - return nil, err - } - } else { - dnsClient = dns.NewClient(dialer, dnsServerAddress) - } - - tel := d.Telemetry() - - dialContext := func(ctx context.Context, network, address string) (net.Conn, error) { - host, port, err := net.SplitHostPort(address) - if err != nil { - return nil, err - } - - if net.ParseIP(host) != nil { - return dialer.DialContext(ctx, network, address) - } - - // Get tracing hooks - trace := httptrace.ContextClientTrace(ctx) - - // Trace DNSStart - ctx, dnsSpan := tel.Tracer().Start(ctx, "keboola.go.apps-proxy.transport.dns.resolve") - dnsSpan.SetAttributes( - attribute.String("transport.dns.resolve.host", host), - attribute.String("transport.dns.resolve.server", dnsClient.DNSServer()), - ) - if trace != nil && trace.DNSStart != nil { - trace.DNSStart(httptrace.DNSStartInfo{Host: host}) - } - - // Create context for DNS resolving - // It separates the events/tracing of the connection to the DNS server, from the connection to the target server. - resolveCtx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), DNSResolveTimeout, errors.New("DNS resolve timeout")) - defer cancel() - - // We are using custom DNS resolving to detect if the target host - data app, is running. - // The DNS query is not recursive and not cached, see "dns" package for details. - ip, err := dnsClient.Resolve(resolveCtx, host) //nolint:contextcheck - - // Trace DNSDone - if trace != nil && trace.DNSDone != nil { - trace.DNSDone(httptrace.DNSDoneInfo{ - Addrs: []net.IPAddr{{IP: net.ParseIP(ip)}}, - Err: err, - }) - } - - // Handle DNS error - dnsSpan.End(&err) - if err != nil { - return nil, err - } - - // Dial - ctx, dialSpan := tel.Tracer().Start(ctx, "keboola.go.apps-proxy.transport.dial") - dialSpan.SetAttributes( - attribute.String("transport.dial.network", network), - attribute.String("transport.dial.ip", ip), - attribute.String("transport.dial.port", port), - ) - conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip, port)) - dialSpan.End(&err) - return conn, err - } - httpTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, - DialContext: dialContext, + DialContext: dialer.DialContext, ForceAttemptHTTP2: true, // HTTP2 is preferred. DisableKeepAlives: false, TLSHandshakeTimeout: TLSHandshakeTimeout,