AP-789: upgrade to Airflow 3.3.0 (#87)

* AP-789: upgrade to Airflow 3.3.0

also moves provider-specific dependencies out of pyproject.toml, etc.

* add updated mokelumne provider packages that work with airflow 3.3.0

* DagBag no longer has an include_examples arg in Airflow 3.3.0

Author: anarchivist

Dockerfile

@@ -1,7 +1,7 @@
 # This Dockerfile relies on ARGs and ENVs defined in the upstream
 # Airflow Dockerfile: https://github.com/apache/airflow/blob/main/Dockerfile
 
-ARG AIRFLOW_VERSION="3.2.2"
+ARG AIRFLOW_VERSION="3.3.0"
 ARG AIRFLOW_IMAGE_NAME="apache/airflow:${AIRFLOW_VERSION}"
 
 FROM ${AIRFLOW_IMAGE_NAME}

constraints.txt

@@ -5,6 +5,5 @@
 botocore == 1.43.0
 cryptography < 49
 fsspec == 2026.4.0
-rich < 14
 flask-appbuilder == 5.2.1
 starlette >= 1.0.1

pyproject.toml

@@ -7,15 +7,13 @@ name = "mokelumne"
 version = "0.7.0"
 dependencies = [
     "apache-airflow-providers-fab>=3.6.1",
-    "apache-airflow-task-sdk == 1.2.2",
+    "apache-airflow-task-sdk == 1.3.0",
     "flask-appbuilder",
     "langchain[aws]",
     "langfuse",
-    "mokelumne-providers-ldc >= 1.0.6",
-    "mokelumne-providers-tind >= 1.0.4",
-    "piffle",  # IIIF API used in batch image description DAG
+    "mokelumne-providers-ldc >= 1.1.0",
+    "mokelumne-providers-tind >= 1.1.0",
     "pymarc",
-    "python-tind-client",
 ]
 requires-python = ">= 3.13, < 3.15"
 authors = [

requirements.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip -n compile pyproject.toml --extra test -c constraints.txt --generate-hashes -o requirements.txt
+#    uv pip compile pyproject.toml --extra test -c constraints.txt --generate-hashes -o requirements.txt
 a2wsgi==1.10.10 \
     --hash=sha256:a5bcffb52081ba39df0d5e9a884fc6f819d92e3a42389343ba77cbf809fe1f45 \
     --hash=sha256:d2b21379479718539dc15fce53b876251a0efe7615352dfe49f6ad1bc507848d
@@ -37,19 +37,19 @@ anyio==4.14.0 \
     #   httpx
     #   starlette
     #   watchfiles
-apache-airflow==3.2.2 \
-    --hash=sha256:5625a596be4a3b96d93d401412bd41a28181ff32ef1219af277ab1d42453b8d6 \
-    --hash=sha256:5fe8f4ca7c930cf2ec5b4f0485074fff8c01f906b6a945ac5add3a760a9aac17
+apache-airflow==3.3.0 \
+    --hash=sha256:59ff9185c4292788dc422d9de5c384ca7b1fc11748ec88a10629d28b3385f3df \
+    --hash=sha256:6e72fc78e60383ba1c807c15f57ea44d46a2f503dff1be5a8dd571c333076303
     # via
     #   apache-airflow-providers-common-compat
     #   apache-airflow-providers-common-io
     #   apache-airflow-providers-common-sql
     #   apache-airflow-providers-fab
     #   apache-airflow-providers-smtp
     #   apache-airflow-providers-standard
-apache-airflow-core==3.2.2 \
-    --hash=sha256:5473699fca8b9b64680d24c6fd2adbf1ad1168facc849cdfaec0f9715d76d7a1 \
-    --hash=sha256:877be429d59193b5e9aab1ae69f44066134bf483c633e9f7c59a9220ebfcc013
+apache-airflow-core==3.3.0 \
+    --hash=sha256:87c0afe38cfbfcb2b8bf91cd2f9af81d053346544b1997abd5794e94697775c1 \
+    --hash=sha256:e58788ae9ad01a55f2fcda5a347eefe09ffef36d1d03d1253b6cd4b5990e91f5
     # via
     #   apache-airflow
     #   apache-airflow-task-sdk
@@ -83,9 +83,9 @@ apache-airflow-providers-standard==1.14.0 \
     --hash=sha256:0119c8b2c54994185a6ebf4e5c706eebf998ad856cc4cc22ffe012de69f328ec \
     --hash=sha256:2fbda76b0809b12abca1dec311062b5f3c7d84453ed502413c2b37e4da4d0543
     # via apache-airflow-core
-apache-airflow-task-sdk==1.2.2 \
-    --hash=sha256:3c8b3c4c86fe97dad5779004218c351cfd85f3dfc5a3f68b86127f266f9b286a \
-    --hash=sha256:709552227b8139b1264413fdc4f0ef3034dd0519c8adeaffd0c9c908a608e6c5
+apache-airflow-task-sdk==1.3.0 \
+    --hash=sha256:13d432b829f958f09bf8dfb20085870886fe60d2646958bde1befa9b018d0406 \
+    --hash=sha256:b4220241e33b80f79113dd21cec414f98baf5028901dcf5ac96156c8fd9fe74c
     # via
     #   mokelumne (pyproject.toml)
     #   apache-airflow
@@ -100,6 +100,10 @@ argcomplete==3.6.3 \
     --hash=sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c \
     --hash=sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce
     # via apache-airflow-core
+arrow==1.4.0 \
+    --hash=sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205 \
+    --hash=sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7
+    # via isoduration
 asgiref==3.11.1 \
     --hash=sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce \
     --hash=sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133
@@ -162,9 +166,9 @@ cachetools==7.1.4 \
     # via
     #   apache-airflow-core
     #   apache-airflow-providers-fab
-cadwyn==7.1.0 \
-    --hash=sha256:0272361a7c97723a9639258a7e1f02c7832b24b7929ac3f3be805eb9de53a550 \
-    --hash=sha256:a72859c326e10dce63cbe360bf0aa88ffd2785be7024b3baafe49be25e2be3d6
+cadwyn==7.0.0 \
+    --hash=sha256:3b57549a37e218dffb55ac5d188639de0516207f05642b084f23627c5a44d614 \
+    --hash=sha256:727d3c444ae992bb2a238246d13f173e4802c92e1c901458975549e6f0522560
     # via apache-airflow-core
 certifi==2026.6.17 \
     --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \
@@ -390,13 +394,14 @@ charset-normalizer==3.4.7 \
     --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \
     --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464
     # via requests
-click==8.4.1 \
-    --hash=sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2 \
-    --hash=sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96
+click==8.4.2 \
+    --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
+    --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
     # via
     #   flask
     #   flask-appbuilder
     #   rich-toolkit
+    #   typer
     #   uvicorn
 colorama==0.4.6 \
     --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
@@ -597,9 +602,9 @@ email-validator==2.3.0 \
     # via
     #   fastapi
     #   flask-appbuilder
-fastapi==0.137.1 \
-    --hash=sha256:64f6983c59e45c4b9fdc44e57cb8035c2451ee91ea8e8ec042aca37de7cf6b69 \
-    --hash=sha256:822360704230d9533d8d9475399613525968aa2f0b5bd2a3ccc9f18c88fd541c
+fastapi==0.136.3 \
+    --hash=sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620 \
+    --hash=sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab
     # via
     #   mokelumne (pyproject.toml)
     #   apache-airflow-core
@@ -907,6 +912,12 @@ iniconfig==2.3.0 \
     --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \
     --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12
     # via pytest
+isoduration==20.11.0 \
+    --hash=sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9 \
+    --hash=sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042
+    # via
+    #   apache-airflow-core
+    #   apache-airflow-task-sdk
 itsdangerous==2.2.0 \
     --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \
     --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173
@@ -974,9 +985,9 @@ langchain-protocol==0.0.17 \
     # via
     #   langchain-core
     #   langgraph-sdk
-langfuse==4.9.0 \
-    --hash=sha256:244d8309cd75d663f29c399f5691e3f04c4a2838e0cc947704e123bdc7233040 \
-    --hash=sha256:ac03eaf7ee6f5fb18036284445833cae92248ae240f3c6068b83d408afb57fe1
+langfuse==4.14.0 \
+    --hash=sha256:31b875c8d09eee39c558584b9424bbd5ed014965c5944c4528a37947ae4b7787 \
+    --hash=sha256:632b74abfa14d9ad3dccbe3322bf7e11b0dce9be0de451b0d134db9bff246d44
     # via mokelumne (pyproject.toml)
 langgraph==1.2.5 \
     --hash=sha256:09a3bdec6fdb3228623fc78b6f69a1400d383f66348d0b04d0efb692022cc6ef \
@@ -1253,13 +1264,13 @@ methodtools==0.4.7 \
     #   apache-airflow-core
     #   apache-airflow-providers-common-sql
     #   apache-airflow-task-sdk
-mokelumne-providers-ldc==1.0.6 \
-    --hash=sha256:76849782b1dd21ea3a739365b0f385635c2ba83d475e8a5c34c36a5cc89e16aa \
-    --hash=sha256:e63b009084ad5d3eaa8cbd11a02b6bcc2ff802d58d7dc8683278cb5ff00cb1eb
+mokelumne-providers-ldc==1.1.0 \
+    --hash=sha256:2eb8da78ce9cffe01ba22716a92681cb7072061abf4be0fad66307bcb74bb44a \
+    --hash=sha256:3e3279bcbf03ee201d3447ae9968c966b6a39390b4240f5bdfa49ae82931e95a
     # via mokelumne (pyproject.toml)
-mokelumne-providers-tind==1.0.4 \
-    --hash=sha256:57dcd5b7cd472e56c39849f75632c9db80354bf7e4b325f66c686844b301f878 \
-    --hash=sha256:e5da892e8e514ded6e7c75fd5b16075b60fe8b79c5d894675a399b5f8361d827
+mokelumne-providers-tind==1.1.0 \
+    --hash=sha256:e3d0a01eadf14e8d4d4db3fef50f35c51af2cc07cc9eb72488da9b7cc294569d \
+    --hash=sha256:e6b985325d565dd6a0c882bcc75a8304bb1fdfaf89d280f989d0138079753f25
     # via mokelumne (pyproject.toml)
 more-itertools==11.1.0 \
     --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \
@@ -1465,54 +1476,55 @@ numpy==2.4.6 \
     --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \
     --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20
     # via langchain-aws
-opentelemetry-api==1.42.1 \
-    --hash=sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714 \
-    --hash=sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716
+opentelemetry-api==1.43.0 \
+    --hash=sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1 \
+    --hash=sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd
     # via
     #   apache-airflow-core
+    #   apache-airflow-task-sdk
     #   langfuse
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
-opentelemetry-exporter-otlp==1.42.1 \
-    --hash=sha256:2d9ebaed714377a67d224d46795ddcc11d2c877fa5de35fda70b6f3b010729a9 \
-    --hash=sha256:aedd54545bb0587cd45210abdc8be545af9c01413f3307786e276df1e3c83bee
+opentelemetry-exporter-otlp==1.43.0 \
+    --hash=sha256:65aded6c50ee7dd2b9948c9d0e59ddb4ed4eea6e8532fba95cbe6a4a64a566ba \
+    --hash=sha256:70f3fe740a64596d4157588a2ee7e4fd37d2acc0c0f522a2882b8c29316cd0f0
     # via apache-airflow-core
-opentelemetry-exporter-otlp-proto-common==1.42.1 \
-    --hash=sha256:04f1f01fb597c4249dfcd7f8b861c902c2102369d376d9d346ff38de4469a2ee \
-    --hash=sha256:f48d395ab815b444da118868977e9798ea354c25737d5cf39578ae894011c140
+opentelemetry-exporter-otlp-proto-common==1.43.0 \
+    --hash=sha256:123c3f9cc87218562490c63b36f497bf3a722faf174a515d1443f31ababa6264 \
+    --hash=sha256:c4e32ba6d6b13bdb2b8f6764c4fd28d00192826561aa04f6d14eedfce7ac076f
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
-opentelemetry-exporter-otlp-proto-grpc==1.42.1 \
-    --hash=sha256:0ae1177e2038b18a929b3098215243631ef91136cba26b7e2b12790ceb7e87cc \
-    --hash=sha256:975c4461f167dd8ed8857d68d3b6b25f3d272eab896f6a9470d0f5b90e2faf15
+opentelemetry-exporter-otlp-proto-grpc==1.43.0 \
+    --hash=sha256:1b3e0627daa9bc21884d4a13946807c255eb558bfe5bdd543dffb6f4c9faee0d \
+    --hash=sha256:6a10d1feacffffda19acacbf277b736094b1e2f4dbb98c90ccb2c6e1962e2ec6
     # via opentelemetry-exporter-otlp
-opentelemetry-exporter-otlp-proto-http==1.42.1 \
-    --hash=sha256:00a16da1b312a1d6c7233d600d557c91df71125af73020f3b9a7765bd699d59d \
-    --hash=sha256:bf142a21035d7571ac3a09cb2e5639f49886f243972883cfe777ed3bf02b734d
+opentelemetry-exporter-otlp-proto-http==1.43.0 \
+    --hash=sha256:647f603aa8efdbdb4dbff842e0729d0406a6fff26b295a72d3d60e7d963b2610 \
+    --hash=sha256:fa8a42bb7d00ee5391f4c0b04d8e6a46c03caa437903296ab73a81dc11ba118f
     # via
     #   langfuse
     #   opentelemetry-exporter-otlp
-opentelemetry-proto==1.42.1 \
-    --hash=sha256:c6a51e6b4f05ae63565f3a113217f3d2bfaec68f78c02d7a6c85f9010d1cfca6 \
-    --hash=sha256:dedb74cba2886c59c7789b227a7a670613025a07489040050aedff6e5c0fb43c
+opentelemetry-proto==1.43.0 \
+    --hash=sha256:224778df17e1f3fafeaaa21d874236ca5f6ffc2f86e0899298ec7351aac27924 \
+    --hash=sha256:c58f1f7ef84bc7dc2834016c0c37fe0081dde7ca9f6339be1970fbf9cdaaa90d
     # via
     #   apache-airflow-core
     #   opentelemetry-exporter-otlp-proto-common
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
-opentelemetry-sdk==1.42.1 \
-    --hash=sha256:083cd4bbfaa5aa7b5a9e552430d9951219967cfb27aa61feb13a77aba1fc839d \
-    --hash=sha256:8c834e8f8c9ba4171d4ec843d0cb8a67e4c7394d3f9e9297e582cbd9456ddbf7
+opentelemetry-sdk==1.43.0 \
+    --hash=sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823 \
+    --hash=sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9
     # via
     #   langfuse
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
-opentelemetry-semantic-conventions==0.63b1 \
-    --hash=sha256:3daf963611334b365e98a57438183eb012d3bfb40b2d931a9af613476b8701a9 \
-    --hash=sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682
+opentelemetry-semantic-conventions==0.64b0 \
+    --hash=sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c \
+    --hash=sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6
     # via opentelemetry-sdk
 ordered-set==4.1.0 \
     --hash=sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562 \
@@ -1747,9 +1759,7 @@ pendulum==3.2.0 \
 piffle==0.9.0 \
     --hash=sha256:5feb8232aa30ab76469f9364b3858342b5c9075aa1af512abe3c6c319dcad2a7 \
     --hash=sha256:c29dc409c37f38763d6fdf6f825711cfcac67284463bf7e606784e4ea496eff2
-    # via
-    #   mokelumne (pyproject.toml)
-    #   mokelumne-providers-tind
+    # via mokelumne-providers-tind
 playwright==1.58.0 \
     --hash=sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117 \
     --hash=sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99 \
@@ -2024,6 +2034,7 @@ python-dateutil==2.9.0.post0 \
     # via
     #   apache-airflow-core
     #   apache-airflow-task-sdk
+    #   arrow
     #   botocore
     #   croniter
     #   flask-appbuilder
@@ -2047,9 +2058,7 @@ python-slugify==8.0.4 \
 python-tind-client==0.2.5 \
     --hash=sha256:9d68d5fc2c914529739ec113977312863cc61ead131ff6529349e795cdadbdaf \
     --hash=sha256:d9a125d36eff3f3b9e5a1c533ec5daa8522661efe3c1919bface304b735cb4a3
-    # via
-    #   mokelumne (pyproject.toml)
-    #   mokelumne-providers-tind
+    # via mokelumne-providers-tind
 pytz==2026.2 \
     --hash=sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126 \
     --hash=sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a
@@ -2178,7 +2187,6 @@ rich==13.9.4 \
     --hash=sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098 \
     --hash=sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90
     # via
-    #   -c constraints.txt
     #   apache-airflow-core
     #   flask-limiter
     #   rich-argparse
@@ -2559,9 +2567,9 @@ text-unidecode==1.3 \
     --hash=sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8 \
     --hash=sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93
     # via python-slugify
-typer==0.26.7 \
-    --hash=sha256:5c87cfbc5d34491c5346ebf49c23e18d56ccb863268d3a8d592b26087c2f5e58 \
-    --hash=sha256:e314a34c617e419c091b2830dda3ea1f257134ff593061a8f5b9717ab8dddb3a
+typer==0.25.1 \
+    --hash=sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89 \
+    --hash=sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc
     # via fastapi-cli
 typing-extensions==4.15.0 \
     --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \
@@ -2602,7 +2610,9 @@ typing-inspection==0.4.2 \
 tzdata==2026.2 \
     --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \
     --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7
-    # via pendulum
+    # via
+    #   arrow
+    #   pendulum
 uc-micro-py==2.0.0 \
     --hash=sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c \
     --hash=sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811

test/dags/test_copy_files.py

@@ -10,7 +10,7 @@
 from mokelumne.dags.copy_files import build_volume_path
 
 DAG_DIR = Path(__file__).resolve().parent.parent.parent / "mokelumne" / "dags"
-_DAG_BAG = DagBag(dag_folder=DAG_DIR.resolve(), include_examples=False)
+_DAG_BAG = DagBag(dag_folder=DAG_DIR.resolve())
 DAG = _DAG_BAG.get_dag("copy_files")
 
 

test/dags/test_gen_llm_image_descriptions.py

@@ -15,7 +15,7 @@
 """The location of the DAGs."""
 
 
-_DAG_BAG = DagBag(dag_folder=DAG_DIR.resolve(), include_examples=False)
+_DAG_BAG = DagBag(dag_folder=DAG_DIR.resolve())
 """The bag of DAGs from which we retrieve our DAG."""
 
 

test/tests/test_dags.py

@@ -10,7 +10,7 @@
 
 @pytest.fixture()
 def dagbag() -> DagBag:
-    return DagBag(dag_folder=dag_dir.resolve(), include_examples=False)
+    return DagBag(dag_folder=dag_dir.resolve())
 
 
 def test_dags_load_with_no_errors(dagbag: DagBag) -> None: